2024-11-19 21:39:47,639 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 21:39:47,657 main DEBUG Took 0.015151 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 21:39:47,657 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 21:39:47,660 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 21:39:47,661 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 21:39:47,662 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,672 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 21:39:47,690 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,694 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,695 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,696 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,697 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,698 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,699 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,699 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,700 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,702 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,702 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,704 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,704 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,705 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,705 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,706 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,707 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,709 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,709 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 21:39:47,710 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,710 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 21:39:47,712 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 21:39:47,714 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 21:39:47,717 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 21:39:47,718 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 21:39:47,721 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 21:39:47,722 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 21:39:47,738 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 21:39:47,742 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 21:39:47,744 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 21:39:47,745 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 21:39:47,746 main DEBUG createAppenders(={Console}) 2024-11-19 21:39:47,747 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 21:39:47,747 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 21:39:47,748 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 21:39:47,749 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 21:39:47,749 main DEBUG OutputStream closed 2024-11-19 21:39:47,749 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 21:39:47,750 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 21:39:47,750 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 21:39:47,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 21:39:47,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 21:39:47,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 21:39:47,855 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 21:39:47,856 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 21:39:47,857 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 21:39:47,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 21:39:47,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 21:39:47,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 21:39:47,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 21:39:47,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 21:39:47,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 21:39:47,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 21:39:47,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 21:39:47,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 21:39:47,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 21:39:47,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 21:39:47,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 21:39:47,878 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 21:39:47,879 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 21:39:47,879 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 21:39:47,881 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T21:39:48,215 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd 2024-11-19 21:39:48,220 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 21:39:48,220 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T21:39:48,234 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-19T21:39:48,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T21:39:48,336 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/cluster_462aee02-848d-d834-50a3-23faa7aec3bf, deleteOnExit=true 2024-11-19T21:39:48,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T21:39:48,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/test.cache.data in system properties and HBase conf 2024-11-19T21:39:48,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T21:39:48,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/hadoop.log.dir in system properties and HBase conf 2024-11-19T21:39:48,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T21:39:48,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T21:39:48,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T21:39:48,470 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T21:39:48,608 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T21:39:48,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T21:39:48,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T21:39:48,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T21:39:48,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T21:39:48,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T21:39:48,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T21:39:48,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T21:39:48,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T21:39:48,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T21:39:48,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/nfs.dump.dir in system properties and HBase conf 2024-11-19T21:39:48,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/java.io.tmpdir in system properties and HBase conf 2024-11-19T21:39:48,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T21:39:48,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T21:39:48,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T21:39:49,894 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T21:39:50,002 INFO [Time-limited test {}] log.Log(170): Logging initialized @3232ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T21:39:50,124 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T21:39:50,231 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T21:39:50,302 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T21:39:50,303 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T21:39:50,305 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T21:39:50,330 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T21:39:50,349 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/hadoop.log.dir/,AVAILABLE} 2024-11-19T21:39:50,350 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T21:39:50,575 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/java.io.tmpdir/jetty-localhost-36811-hadoop-hdfs-3_4_1-tests_jar-_-any-932375514632832913/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T21:39:50,582 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:36811} 2024-11-19T21:39:50,583 INFO [Time-limited test {}] server.Server(415): Started @3814ms 2024-11-19T21:39:51,196 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T21:39:51,207 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T21:39:51,211 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T21:39:51,211 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T21:39:51,211 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T21:39:51,212 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ff64ba0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/hadoop.log.dir/,AVAILABLE} 2024-11-19T21:39:51,213 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4828e07b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T21:39:51,329 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@51084f0f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/java.io.tmpdir/jetty-localhost-45521-hadoop-hdfs-3_4_1-tests_jar-_-any-10601860196528012405/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T21:39:51,330 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@370621d9{HTTP/1.1, (http/1.1)}{localhost:45521} 2024-11-19T21:39:51,330 INFO [Time-limited test {}] server.Server(415): Started @4561ms 2024-11-19T21:39:51,378 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T21:39:52,195 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/cluster_462aee02-848d-d834-50a3-23faa7aec3bf/data/data2/current/BP-224541545-172.17.0.2-1732052389373/current, will proceed with Du for space computation calculation, 2024-11-19T21:39:52,195 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/cluster_462aee02-848d-d834-50a3-23faa7aec3bf/data/data1/current/BP-224541545-172.17.0.2-1732052389373/current, will proceed with Du for space computation calculation, 2024-11-19T21:39:52,225 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T21:39:52,269 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x59020f0a361e8ef8 with lease ID 0xb7e72cbe0a82caa: Processing first storage report for DS-b97bc27d-459d-4d67-90c6-67062de71bdf from datanode DatanodeRegistration(127.0.0.1:38071, datanodeUuid=805e117f-2c48-4e99-9963-f94e586ba0fe, infoPort=38077, infoSecurePort=0, ipcPort=42583, storageInfo=lv=-57;cid=testClusterID;nsid=963582697;c=1732052389373) 2024-11-19T21:39:52,270 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x59020f0a361e8ef8 with lease ID 0xb7e72cbe0a82caa: from storage DS-b97bc27d-459d-4d67-90c6-67062de71bdf node DatanodeRegistration(127.0.0.1:38071, datanodeUuid=805e117f-2c48-4e99-9963-f94e586ba0fe, infoPort=38077, infoSecurePort=0, ipcPort=42583, storageInfo=lv=-57;cid=testClusterID;nsid=963582697;c=1732052389373), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T21:39:52,270 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x59020f0a361e8ef8 with lease ID 0xb7e72cbe0a82caa: Processing first storage report for DS-ee6c1636-c722-48d2-be57-209703869357 from datanode DatanodeRegistration(127.0.0.1:38071, datanodeUuid=805e117f-2c48-4e99-9963-f94e586ba0fe, infoPort=38077, infoSecurePort=0, ipcPort=42583, storageInfo=lv=-57;cid=testClusterID;nsid=963582697;c=1732052389373) 2024-11-19T21:39:52,270 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x59020f0a361e8ef8 with lease ID 0xb7e72cbe0a82caa: from storage DS-ee6c1636-c722-48d2-be57-209703869357 node DatanodeRegistration(127.0.0.1:38071, datanodeUuid=805e117f-2c48-4e99-9963-f94e586ba0fe, infoPort=38077, infoSecurePort=0, ipcPort=42583, storageInfo=lv=-57;cid=testClusterID;nsid=963582697;c=1732052389373), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T21:39:52,279 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd 2024-11-19T21:39:52,346 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/cluster_462aee02-848d-d834-50a3-23faa7aec3bf/zookeeper_0, clientPort=63776, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/cluster_462aee02-848d-d834-50a3-23faa7aec3bf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/cluster_462aee02-848d-d834-50a3-23faa7aec3bf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T21:39:52,356 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63776 2024-11-19T21:39:52,367 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T21:39:52,370 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T21:39:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741825_1001 (size=7) 2024-11-19T21:39:53,000 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b with version=8 2024-11-19T21:39:53,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/hbase-staging 2024-11-19T21:39:53,080 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T21:39:53,342 INFO [Time-limited test {}] client.ConnectionUtils(128): master/707262ffa86d:0 server-side Connection retries=45 2024-11-19T21:39:53,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:39:53,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T21:39:53,360 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T21:39:53,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:39:53,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T21:39:53,527 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T21:39:53,595 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T21:39:53,604 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T21:39:53,608 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T21:39:53,633 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1534 (auto-detected) 2024-11-19T21:39:53,634 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T21:39:53,650 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37541 2024-11-19T21:39:53,670 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37541 connecting to ZooKeeper ensemble=127.0.0.1:63776 2024-11-19T21:39:53,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:375410x0, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T21:39:53,766 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37541-0x1015653ac4f0000 connected 2024-11-19T21:39:53,860 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T21:39:53,862 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T21:39:53,873 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:39:53,878 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b, hbase.cluster.distributed=false 2024-11-19T21:39:53,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T21:39:53,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37541 2024-11-19T21:39:53,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37541 2024-11-19T21:39:53,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37541 2024-11-19T21:39:53,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37541 2024-11-19T21:39:53,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37541 2024-11-19T21:39:54,046 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/707262ffa86d:0 server-side Connection retries=45 2024-11-19T21:39:54,048 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:39:54,048 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T21:39:54,049 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T21:39:54,049 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T21:39:54,049 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T21:39:54,051 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T21:39:54,054 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T21:39:54,055 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42821 2024-11-19T21:39:54,057 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42821 connecting to ZooKeeper ensemble=127.0.0.1:63776 2024-11-19T21:39:54,058 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T21:39:54,062 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T21:39:54,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428210x0, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T21:39:54,100 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:428210x0, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:39:54,100 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42821-0x1015653ac4f0001 connected 2024-11-19T21:39:54,105 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T21:39:54,116 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T21:39:54,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T21:39:54,127 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T21:39:54,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42821 2024-11-19T21:39:54,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42821 2024-11-19T21:39:54,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42821 2024-11-19T21:39:54,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42821 2024-11-19T21:39:54,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42821 2024-11-19T21:39:54,158 DEBUG [M:0;707262ffa86d:37541 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;707262ffa86d:37541 2024-11-19T21:39:54,160 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/707262ffa86d,37541,1732052393169 2024-11-19T21:39:54,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:39:54,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:39:54,181 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/707262ffa86d,37541,1732052393169 2024-11-19T21:39:54,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:54,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T21:39:54,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:54,209 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T21:39:54,210 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/707262ffa86d,37541,1732052393169 from backup master directory 2024-11-19T21:39:54,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:39:54,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/707262ffa86d,37541,1732052393169 2024-11-19T21:39:54,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T21:39:54,221 WARN [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T21:39:54,221 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=707262ffa86d,37541,1732052393169 2024-11-19T21:39:54,223 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T21:39:54,225 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T21:39:54,280 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/hbase.id] with ID: 18a7bc1d-40c6-490c-9d94-197e3bb518aa 2024-11-19T21:39:54,280 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/.tmp/hbase.id 2024-11-19T21:39:54,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741826_1002 (size=42) 2024-11-19T21:39:54,696 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/.tmp/hbase.id]:[hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/hbase.id] 2024-11-19T21:39:54,752 INFO [master/707262ffa86d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T21:39:54,759 INFO [master/707262ffa86d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T21:39:54,777 INFO [master/707262ffa86d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-19T21:39:54,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:54,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:54,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741827_1003 (size=196) 2024-11-19T21:39:55,274 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:39:55,276 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T21:39:55,290 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:39:55,293 INFO [master/707262ffa86d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:39:55,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741828_1004 (size=1189) 2024-11-19T21:39:55,747 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store 2024-11-19T21:39:55,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741829_1005 (size=34) 2024-11-19T21:39:56,177 INFO [master/707262ffa86d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T21:39:56,181 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:39:56,183 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T21:39:56,183 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:39:56,184 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:39:56,186 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T21:39:56,187 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:39:56,187 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:39:56,189 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732052396183Disabling compacts and flushes for region at 1732052396183Disabling writes for close at 1732052396187 (+4 ms)Writing region close event to WAL at 1732052396187Closed at 1732052396187 2024-11-19T21:39:56,192 WARN [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/.initializing 2024-11-19T21:39:56,192 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/WALs/707262ffa86d,37541,1732052393169 2024-11-19T21:39:56,202 INFO [master/707262ffa86d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T21:39:56,222 INFO [master/707262ffa86d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=707262ffa86d%2C37541%2C1732052393169, suffix=, logDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/WALs/707262ffa86d,37541,1732052393169, archiveDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/oldWALs, maxLogs=10 2024-11-19T21:39:56,246 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/WALs/707262ffa86d,37541,1732052393169/707262ffa86d%2C37541%2C1732052393169.1732052396227, exclude list is [], retry=0 2024-11-19T21:39:56,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b97bc27d-459d-4d67-90c6-67062de71bdf,DISK] 2024-11-19T21:39:56,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T21:39:56,309 INFO [master/707262ffa86d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/WALs/707262ffa86d,37541,1732052393169/707262ffa86d%2C37541%2C1732052393169.1732052396227 2024-11-19T21:39:56,310 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38077:38077)] 2024-11-19T21:39:56,310 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:39:56,311 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:39:56,315 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,316 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,355 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T21:39:56,386 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:56,389 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:39:56,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T21:39:56,394 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:56,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:39:56,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,399 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T21:39:56,399 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:56,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:39:56,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,405 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T21:39:56,405 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:56,406 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:39:56,407 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,412 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,414 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,421 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,422 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,427 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T21:39:56,432 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T21:39:56,436 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:39:56,438 INFO [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62243336, jitterRate=-0.0725020170211792}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T21:39:56,445 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732052396330Initializing all the Stores at 1732052396333 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732052396333Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052396334 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052396335 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052396335Cleaning up temporary data from old regions at 1732052396423 (+88 ms)Region opened successfully at 1732052396445 (+22 ms) 2024-11-19T21:39:56,447 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T21:39:56,485 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491b3393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=707262ffa86d/172.17.0.2:0 2024-11-19T21:39:56,518 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T21:39:56,532 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T21:39:56,532 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T21:39:56,536 INFO [master/707262ffa86d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T21:39:56,538 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T21:39:56,545 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-19T21:39:56,545 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T21:39:56,586 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T21:39:56,598 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T21:39:56,640 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T21:39:56,644 INFO [master/707262ffa86d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T21:39:56,647 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T21:39:56,657 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T21:39:56,661 INFO [master/707262ffa86d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T21:39:56,666 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T21:39:56,677 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T21:39:56,680 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T21:39:56,690 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T21:39:56,715 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T21:39:56,727 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T21:39:56,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T21:39:56,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T21:39:56,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:56,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:56,744 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=707262ffa86d,37541,1732052393169, sessionid=0x1015653ac4f0000, setting cluster-up flag (Was=false) 2024-11-19T21:39:56,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:56,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:56,802 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T21:39:56,805 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=707262ffa86d,37541,1732052393169 2024-11-19T21:39:56,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:56,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:56,852 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T21:39:56,855 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=707262ffa86d,37541,1732052393169 2024-11-19T21:39:56,861 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T21:39:56,929 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T21:39:56,938 INFO [master/707262ffa86d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T21:39:56,944 INFO [master/707262ffa86d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T21:39:56,948 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(746): ClusterId : 18a7bc1d-40c6-490c-9d94-197e3bb518aa 2024-11-19T21:39:56,951 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T21:39:56,950 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 707262ffa86d,37541,1732052393169 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T21:39:56,988 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T21:39:56,988 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T21:39:56,988 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:39:56,988 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:39:56,989 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:39:56,989 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/707262ffa86d:0, corePoolSize=5, maxPoolSize=5 2024-11-19T21:39:56,989 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/707262ffa86d:0, corePoolSize=10, maxPoolSize=10 2024-11-19T21:39:56,989 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:56,989 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/707262ffa86d:0, corePoolSize=2, maxPoolSize=2 2024-11-19T21:39:56,989 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:56,994 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732052426993 2024-11-19T21:39:56,995 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T21:39:56,995 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T21:39:56,996 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T21:39:56,996 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T21:39:57,000 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T21:39:57,000 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T21:39:57,000 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T21:39:57,001 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T21:39:57,001 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T21:39:57,001 DEBUG [RS:0;707262ffa86d:42821 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd221cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=707262ffa86d/172.17.0.2:0 2024-11-19T21:39:57,002 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:57,003 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T21:39:57,002 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,011 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T21:39:57,012 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T21:39:57,013 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T21:39:57,015 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T21:39:57,016 INFO [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T21:39:57,018 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;707262ffa86d:42821 2024-11-19T21:39:57,018 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.large.0-1732052397017,5,FailOnTimeoutGroup] 2024-11-19T21:39:57,021 INFO [RS:0;707262ffa86d:42821 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T21:39:57,022 INFO [RS:0;707262ffa86d:42821 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T21:39:57,022 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T21:39:57,022 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.small.0-1732052397018,5,FailOnTimeoutGroup] 2024-11-19T21:39:57,022 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,023 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T21:39:57,024 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,024 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(2659): reportForDuty to master=707262ffa86d,37541,1732052393169 with port=42821, startcode=1732052394005 2024-11-19T21:39:57,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741831_1007 (size=1321) 2024-11-19T21:39:57,025 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,035 DEBUG [RS:0;707262ffa86d:42821 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T21:39:57,063 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59845, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T21:39:57,069 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37541 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 707262ffa86d,42821,1732052394005 2024-11-19T21:39:57,071 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37541 {}] master.ServerManager(517): Registering regionserver=707262ffa86d,42821,1732052394005 2024-11-19T21:39:57,083 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b 2024-11-19T21:39:57,083 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37961 2024-11-19T21:39:57,083 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T21:39:57,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T21:39:57,125 DEBUG [RS:0;707262ffa86d:42821 {}] zookeeper.ZKUtil(111): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/707262ffa86d,42821,1732052394005 2024-11-19T21:39:57,125 WARN [RS:0;707262ffa86d:42821 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T21:39:57,125 INFO [RS:0;707262ffa86d:42821 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:39:57,125 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/WALs/707262ffa86d,42821,1732052394005 2024-11-19T21:39:57,128 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [707262ffa86d,42821,1732052394005] 2024-11-19T21:39:57,157 INFO [RS:0;707262ffa86d:42821 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T21:39:57,172 INFO [RS:0;707262ffa86d:42821 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T21:39:57,177 INFO [RS:0;707262ffa86d:42821 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T21:39:57,177 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,178 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T21:39:57,183 INFO [RS:0;707262ffa86d:42821 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T21:39:57,184 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,184 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,184 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,184 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,185 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,185 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,185 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/707262ffa86d:0, corePoolSize=2, maxPoolSize=2 2024-11-19T21:39:57,185 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,185 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,185 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,185 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,186 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,186 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/707262ffa86d:0, corePoolSize=1, maxPoolSize=1 2024-11-19T21:39:57,186 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:39:57,186 DEBUG [RS:0;707262ffa86d:42821 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0, corePoolSize=3, maxPoolSize=3 2024-11-19T21:39:57,187 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,187 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,187 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,188 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,188 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,188 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,42821,1732052394005-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T21:39:57,207 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T21:39:57,208 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,42821,1732052394005-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,209 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,209 INFO [RS:0;707262ffa86d:42821 {}] regionserver.Replication(171): 707262ffa86d,42821,1732052394005 started 2024-11-19T21:39:57,230 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:57,231 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1482): Serving as 707262ffa86d,42821,1732052394005, RpcServer on 707262ffa86d/172.17.0.2:42821, sessionid=0x1015653ac4f0001 2024-11-19T21:39:57,231 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T21:39:57,232 DEBUG [RS:0;707262ffa86d:42821 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 707262ffa86d,42821,1732052394005 2024-11-19T21:39:57,232 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,42821,1732052394005' 2024-11-19T21:39:57,232 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T21:39:57,233 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T21:39:57,234 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T21:39:57,234 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T21:39:57,235 DEBUG [RS:0;707262ffa86d:42821 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 707262ffa86d,42821,1732052394005 2024-11-19T21:39:57,235 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '707262ffa86d,42821,1732052394005' 2024-11-19T21:39:57,235 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T21:39:57,236 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T21:39:57,237 DEBUG [RS:0;707262ffa86d:42821 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T21:39:57,237 INFO [RS:0;707262ffa86d:42821 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T21:39:57,237 INFO [RS:0;707262ffa86d:42821 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T21:39:57,344 INFO [RS:0;707262ffa86d:42821 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T21:39:57,348 INFO [RS:0;707262ffa86d:42821 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=707262ffa86d%2C42821%2C1732052394005, suffix=, logDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/WALs/707262ffa86d,42821,1732052394005, archiveDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/oldWALs, maxLogs=32 2024-11-19T21:39:57,364 DEBUG [RS:0;707262ffa86d:42821 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/WALs/707262ffa86d,42821,1732052394005/707262ffa86d%2C42821%2C1732052394005.1732052397351, exclude list is [], retry=0 2024-11-19T21:39:57,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b97bc27d-459d-4d67-90c6-67062de71bdf,DISK] 2024-11-19T21:39:57,371 INFO [RS:0;707262ffa86d:42821 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/WALs/707262ffa86d,42821,1732052394005/707262ffa86d%2C42821%2C1732052394005.1732052397351 2024-11-19T21:39:57,372 DEBUG [RS:0;707262ffa86d:42821 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38077:38077)] 2024-11-19T21:39:57,428 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T21:39:57,429 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b 2024-11-19T21:39:57,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741833_1009 (size=32) 2024-11-19T21:39:57,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:39:57,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T21:39:57,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T21:39:57,861 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:57,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:39:57,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T21:39:57,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T21:39:57,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:57,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:39:57,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T21:39:57,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T21:39:57,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:57,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:39:57,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T21:39:57,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T21:39:57,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:57,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:39:57,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T21:39:57,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740 2024-11-19T21:39:57,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740 2024-11-19T21:39:57,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T21:39:57,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T21:39:57,879 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:39:57,881 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T21:39:57,885 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:39:57,886 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64700795, jitterRate=-0.03588302433490753}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:39:57,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732052397850Initializing all the Stores at 1732052397856 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732052397856Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732052397856Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052397856Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732052397857 (+1 ms)Cleaning up temporary data from old regions at 1732052397878 (+21 ms)Region opened successfully at 1732052397889 (+11 ms) 2024-11-19T21:39:57,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T21:39:57,889 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T21:39:57,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T21:39:57,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T21:39:57,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T21:39:57,891 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T21:39:57,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732052397889Disabling compacts and flushes for region at 1732052397889Disabling writes for close at 1732052397890 (+1 ms)Writing region close event to WAL at 1732052397890Closed at 1732052397891 (+1 ms) 2024-11-19T21:39:57,894 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T21:39:57,894 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T21:39:57,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T21:39:57,907 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T21:39:57,909 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T21:39:58,064 DEBUG [707262ffa86d:37541 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T21:39:58,077 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:39:58,082 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 707262ffa86d,42821,1732052394005, state=OPENING 2024-11-19T21:39:58,140 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T21:39:58,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:58,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:39:58,154 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:39:58,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:39:58,158 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T21:39:58,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:39:58,339 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T21:39:58,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48867, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T21:39:58,352 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T21:39:58,352 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T21:39:58,353 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T21:39:58,357 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=707262ffa86d%2C42821%2C1732052394005.meta, suffix=.meta, logDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/WALs/707262ffa86d,42821,1732052394005, archiveDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/oldWALs, maxLogs=32 2024-11-19T21:39:58,372 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/WALs/707262ffa86d,42821,1732052394005/707262ffa86d%2C42821%2C1732052394005.meta.1732052398359.meta, exclude list is [], retry=0 2024-11-19T21:39:58,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b97bc27d-459d-4d67-90c6-67062de71bdf,DISK] 2024-11-19T21:39:58,379 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/WALs/707262ffa86d,42821,1732052394005/707262ffa86d%2C42821%2C1732052394005.meta.1732052398359.meta 2024-11-19T21:39:58,379 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38077:38077)] 2024-11-19T21:39:58,380 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:39:58,382 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T21:39:58,385 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T21:39:58,389 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T21:39:58,394 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T21:39:58,394 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:39:58,395 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T21:39:58,395 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T21:39:58,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T21:39:58,400 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T21:39:58,400 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:58,401 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:39:58,401 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T21:39:58,402 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T21:39:58,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:58,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:39:58,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T21:39:58,405 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T21:39:58,405 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:58,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:39:58,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T21:39:58,408 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T21:39:58,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:58,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T21:39:58,409 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T21:39:58,411 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740 2024-11-19T21:39:58,414 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740 2024-11-19T21:39:58,416 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T21:39:58,416 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T21:39:58,417 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:39:58,420 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T21:39:58,422 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60035398, jitterRate=-0.1054028570652008}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:39:58,422 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T21:39:58,424 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732052398395Writing region info on filesystem at 1732052398396 (+1 ms)Initializing all the Stores at 1732052398397 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732052398398 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732052398398Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052398398Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732052398398Cleaning up temporary data from old regions at 1732052398416 (+18 ms)Running coprocessor post-open hooks at 1732052398422 (+6 ms)Region opened successfully at 1732052398424 (+2 ms) 2024-11-19T21:39:58,431 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732052398331 2024-11-19T21:39:58,442 DEBUG [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T21:39:58,443 INFO [RS_OPEN_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T21:39:58,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:39:58,448 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 707262ffa86d,42821,1732052394005, state=OPEN 2024-11-19T21:39:58,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T21:39:58,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T21:39:58,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:39:58,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T21:39:58,502 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=707262ffa86d,42821,1732052394005 2024-11-19T21:39:58,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T21:39:58,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=707262ffa86d,42821,1732052394005 in 341 msec 2024-11-19T21:39:58,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T21:39:58,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 610 msec 2024-11-19T21:39:58,517 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T21:39:58,518 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T21:39:58,537 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:39:58,538 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:39:58,557 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:39:58,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43047, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:39:58,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6880 sec 2024-11-19T21:39:58,580 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732052398580, completionTime=-1 2024-11-19T21:39:58,583 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T21:39:58,583 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T21:39:58,607 INFO [master/707262ffa86d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-19T21:39:58,607 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732052458607 2024-11-19T21:39:58,607 INFO [master/707262ffa86d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732052518607 2024-11-19T21:39:58,607 INFO [master/707262ffa86d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-19T21:39:58,609 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,37541,1732052393169-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:58,609 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,37541,1732052393169-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:58,610 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,37541,1732052393169-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:58,611 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-707262ffa86d:37541, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:58,611 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:58,612 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:58,617 DEBUG [master/707262ffa86d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T21:39:58,665 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.443sec 2024-11-19T21:39:58,666 INFO [master/707262ffa86d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T21:39:58,668 INFO [master/707262ffa86d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T21:39:58,669 INFO [master/707262ffa86d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T21:39:58,670 INFO [master/707262ffa86d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T21:39:58,670 INFO [master/707262ffa86d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T21:39:58,671 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,37541,1732052393169-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T21:39:58,672 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,37541,1732052393169-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T21:39:58,682 DEBUG [master/707262ffa86d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T21:39:58,683 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T21:39:58,683 INFO [master/707262ffa86d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=707262ffa86d,37541,1732052393169-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T21:39:58,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3e3753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:39:58,759 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T21:39:58,759 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T21:39:58,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:39:58,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:39:58,772 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:39:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:39:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:39:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a49088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:39:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:39:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:39:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:39:58,781 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:39:58,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f487a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:39:58,785 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:39:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:39:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:39:58,795 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:39:58,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=707262ffa86d,37541,1732052393169 2024-11-19T21:39:58,826 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=305, ProcessCount=11, AvailableMemoryMB=5838 2024-11-19T21:39:58,833 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T21:39:58,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 707262ffa86d,37541,1732052393169 2024-11-19T21:39:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@49a86ad1 2024-11-19T21:39:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T21:39:58,844 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T21:39:58,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T21:39:58,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:39:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T21:39:58,866 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T21:39:58,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T21:39:58,869 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:39:58,872 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T21:39:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:39:58,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741835_1011 (size=959) 2024-11-19T21:39:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:39:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:39:59,291 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b 2024-11-19T21:39:59,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741836_1012 (size=53) 2024-11-19T21:39:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:39:59,703 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:39:59,703 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7f292f8e50ece577fd7cb587fbc66fc6, disabling compactions & flushes 2024-11-19T21:39:59,703 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:39:59,704 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:39:59,704 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. after waiting 0 ms 2024-11-19T21:39:59,704 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:39:59,704 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:39:59,704 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7f292f8e50ece577fd7cb587fbc66fc6: Waiting for close lock at 1732052399703Disabling compacts and flushes for region at 1732052399703Disabling writes for close at 1732052399704 (+1 ms)Writing region close event to WAL at 1732052399704Closed at 1732052399704 2024-11-19T21:39:59,710 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T21:39:59,718 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732052399711"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732052399711"}]},"ts":"1732052399711"} 2024-11-19T21:39:59,722 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T21:39:59,724 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T21:39:59,726 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052399724"}]},"ts":"1732052399724"} 2024-11-19T21:39:59,731 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T21:39:59,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f292f8e50ece577fd7cb587fbc66fc6, ASSIGN}] 2024-11-19T21:39:59,735 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f292f8e50ece577fd7cb587fbc66fc6, ASSIGN 2024-11-19T21:39:59,737 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f292f8e50ece577fd7cb587fbc66fc6, ASSIGN; state=OFFLINE, location=707262ffa86d,42821,1732052394005; forceNewPlan=false, retain=false 2024-11-19T21:39:59,891 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7f292f8e50ece577fd7cb587fbc66fc6, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:39:59,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f292f8e50ece577fd7cb587fbc66fc6, ASSIGN because future has completed 2024-11-19T21:39:59,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:40:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:40:00,066 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:00,066 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:40:00,067 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,068 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:40:00,068 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,068 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,072 INFO [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,074 INFO [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f292f8e50ece577fd7cb587fbc66fc6 columnFamilyName A 2024-11-19T21:40:00,074 DEBUG [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:00,076 INFO [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] regionserver.HStore(327): Store=7f292f8e50ece577fd7cb587fbc66fc6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:00,076 INFO [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,078 INFO [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f292f8e50ece577fd7cb587fbc66fc6 columnFamilyName B 2024-11-19T21:40:00,078 DEBUG [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:00,079 INFO [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] regionserver.HStore(327): Store=7f292f8e50ece577fd7cb587fbc66fc6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:00,080 INFO [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,082 INFO [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f292f8e50ece577fd7cb587fbc66fc6 columnFamilyName C 2024-11-19T21:40:00,082 DEBUG [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:00,083 INFO [StoreOpener-7f292f8e50ece577fd7cb587fbc66fc6-1 {}] regionserver.HStore(327): Store=7f292f8e50ece577fd7cb587fbc66fc6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:00,083 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,085 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,085 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,087 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,087 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,088 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:40:00,090 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,094 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:40:00,095 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7f292f8e50ece577fd7cb587fbc66fc6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69061039, jitterRate=0.02908967435359955}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:40:00,095 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:00,096 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7f292f8e50ece577fd7cb587fbc66fc6: Running coprocessor pre-open hook at 1732052400068Writing region info on filesystem at 1732052400068Initializing all the Stores at 1732052400071 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052400071Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052400071Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052400071Cleaning up temporary data from old regions at 1732052400087 (+16 ms)Running coprocessor post-open hooks at 1732052400095 (+8 ms)Region opened successfully at 1732052400096 (+1 ms) 2024-11-19T21:40:00,098 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., pid=6, masterSystemTime=1732052400056 2024-11-19T21:40:00,101 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:00,101 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:00,102 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7f292f8e50ece577fd7cb587fbc66fc6, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:00,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:40:00,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T21:40:00,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 in 206 msec 2024-11-19T21:40:00,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T21:40:00,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f292f8e50ece577fd7cb587fbc66fc6, ASSIGN in 381 msec 2024-11-19T21:40:00,119 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T21:40:00,120 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052400119"}]},"ts":"1732052400119"} 2024-11-19T21:40:00,126 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T21:40:00,128 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T21:40:00,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2720 sec 2024-11-19T21:40:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T21:40:01,042 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:01,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed391d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,056 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dee243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,059 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5efef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,067 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T21:40:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 707262ffa86d,37541,1732052393169 2024-11-19T21:40:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f48350e 2024-11-19T21:40:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T21:40:01,073 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T21:40:01,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6c3037, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,077 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,077 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,079 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,081 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a519624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,085 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2555cfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,088 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ab23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa662a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,090 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,090 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,092 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,096 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5a580f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a38e5f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,100 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1613decc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,103 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f588bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658d066f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,108 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c9f602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,111 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1853672d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d220de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,115 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562d1bb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,118 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31d0d17a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32cda1eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,121 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf634c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,124 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b9987f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d59c73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,126 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,126 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,127 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c44f32e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,129 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306cf84f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27451c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:01,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:01,136 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:01,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c17951c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:01,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:01,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:01,138 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315b49bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:01,142 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,143 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,145 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,145 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,147 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,147 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,148 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:01,152 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,152 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,154 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,157 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:40:01,158 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:40:01,159 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:40:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:40:01,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:40:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:40:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T21:40:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:40:01,166 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T21:40:01,169 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:01,172 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:01,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:01,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:01,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:40:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:40:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:01,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:01,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:40:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T21:40:01,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T21:40:01,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:01,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/c537bd317d784ed69e961bcb357b8246 is 50, key is test_row_0/A:col10/1732052401243/Put/seqid=0 2024-11-19T21:40:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:01,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:01,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:01,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741837_1013 (size=14341) 2024-11-19T21:40:01,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/c537bd317d784ed69e961bcb357b8246 2024-11-19T21:40:01,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052461409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052461403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052461416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052461425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052461423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T21:40:01,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T21:40:01,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:01,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:01,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:01,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/4489f949efd841edb251a3032aa04c49 is 50, key is test_row_0/B:col10/1732052401243/Put/seqid=0 2024-11-19T21:40:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052461590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052461591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052461592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052461591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052461592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741838_1014 (size=12001) 2024-11-19T21:40:01,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/4489f949efd841edb251a3032aa04c49 2024-11-19T21:40:01,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3d00bd83ba0a42bda1b7ee504d109a28 is 50, key is test_row_0/C:col10/1732052401243/Put/seqid=0 2024-11-19T21:40:01,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T21:40:01,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741839_1015 (size=12001) 2024-11-19T21:40:01,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3d00bd83ba0a42bda1b7ee504d109a28 2024-11-19T21:40:01,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:01,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:01,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:01,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:01,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:01,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/c537bd317d784ed69e961bcb357b8246 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/c537bd317d784ed69e961bcb357b8246 2024-11-19T21:40:01,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/c537bd317d784ed69e961bcb357b8246, entries=200, sequenceid=13, filesize=14.0 K 2024-11-19T21:40:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T21:40:01,801 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T21:40:01,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/4489f949efd841edb251a3032aa04c49 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4489f949efd841edb251a3032aa04c49 2024-11-19T21:40:01,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052461800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052461809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052461810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052461811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052461811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:01,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4489f949efd841edb251a3032aa04c49, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T21:40:01,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3d00bd83ba0a42bda1b7ee504d109a28 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d00bd83ba0a42bda1b7ee504d109a28 2024-11-19T21:40:01,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d00bd83ba0a42bda1b7ee504d109a28, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T21:40:01,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 7f292f8e50ece577fd7cb587fbc66fc6 in 624ms, sequenceid=13, compaction requested=false 2024-11-19T21:40:01,872 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T21:40:01,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:01,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T21:40:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:01,909 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:40:01,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e7831ae6c590489e8425ca9944c733b9 is 50, key is test_row_0/A:col10/1732052401401/Put/seqid=0 2024-11-19T21:40:01,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741840_1016 (size=12001) 2024-11-19T21:40:01,960 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e7831ae6c590489e8425ca9944c733b9 2024-11-19T21:40:01,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/a3d196969853431b9eaaf7e86ebb1381 is 50, key is test_row_0/B:col10/1732052401401/Put/seqid=0 2024-11-19T21:40:02,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741841_1017 (size=12001) 2024-11-19T21:40:02,020 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/a3d196969853431b9eaaf7e86ebb1381 2024-11-19T21:40:02,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/58a01d0e759f4c7f995fd7dd328b63cb is 50, key is test_row_0/C:col10/1732052401401/Put/seqid=0 2024-11-19T21:40:02,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741842_1018 (size=12001) 2024-11-19T21:40:02,109 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/58a01d0e759f4c7f995fd7dd328b63cb 2024-11-19T21:40:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:02,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:02,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e7831ae6c590489e8425ca9944c733b9 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e7831ae6c590489e8425ca9944c733b9 2024-11-19T21:40:02,155 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e7831ae6c590489e8425ca9944c733b9, entries=150, sequenceid=38, filesize=11.7 K 2024-11-19T21:40:02,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/a3d196969853431b9eaaf7e86ebb1381 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a3d196969853431b9eaaf7e86ebb1381 2024-11-19T21:40:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052462157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052462161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052462167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052462169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052462173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,193 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a3d196969853431b9eaaf7e86ebb1381, entries=150, sequenceid=38, filesize=11.7 K 2024-11-19T21:40:02,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/58a01d0e759f4c7f995fd7dd328b63cb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/58a01d0e759f4c7f995fd7dd328b63cb 2024-11-19T21:40:02,211 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/58a01d0e759f4c7f995fd7dd328b63cb, entries=150, sequenceid=38, filesize=11.7 K 2024-11-19T21:40:02,214 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7f292f8e50ece577fd7cb587fbc66fc6 in 304ms, sequenceid=38, compaction requested=false 2024-11-19T21:40:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T21:40:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T21:40:02,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T21:40:02,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0450 sec 2024-11-19T21:40:02,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.0780 sec 2024-11-19T21:40:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:02,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:02,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:40:02,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/8f420f4e75384407a60d727f7ed2194a is 50, key is test_row_0/A:col10/1732052402166/Put/seqid=0 2024-11-19T21:40:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T21:40:02,310 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:02,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T21:40:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T21:40:02,332 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:02,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741843_1019 (size=14341) 2024-11-19T21:40:02,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/8f420f4e75384407a60d727f7ed2194a 2024-11-19T21:40:02,338 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:02,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:02,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/3459970beb454c7fa24792d489633a5a is 50, key is test_row_0/B:col10/1732052402166/Put/seqid=0 2024-11-19T21:40:02,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741844_1020 (size=12001) 2024-11-19T21:40:02,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/3459970beb454c7fa24792d489633a5a 2024-11-19T21:40:02,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052462388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052462401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052462405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052462407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052462408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/35a5a06e068e4fb4831cff45715d150c is 50, key is test_row_0/C:col10/1732052402166/Put/seqid=0 2024-11-19T21:40:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T21:40:02,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741845_1021 (size=12001) 2024-11-19T21:40:02,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/35a5a06e068e4fb4831cff45715d150c 2024-11-19T21:40:02,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/8f420f4e75384407a60d727f7ed2194a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8f420f4e75384407a60d727f7ed2194a 2024-11-19T21:40:02,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8f420f4e75384407a60d727f7ed2194a, entries=200, sequenceid=50, filesize=14.0 K 2024-11-19T21:40:02,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/3459970beb454c7fa24792d489633a5a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3459970beb454c7fa24792d489633a5a 2024-11-19T21:40:02,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T21:40:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3459970beb454c7fa24792d489633a5a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T21:40:02,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/35a5a06e068e4fb4831cff45715d150c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/35a5a06e068e4fb4831cff45715d150c 2024-11-19T21:40:02,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052462521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052462522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052462522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/35a5a06e068e4fb4831cff45715d150c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T21:40:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 7f292f8e50ece577fd7cb587fbc66fc6 in 247ms, sequenceid=50, compaction requested=true 2024-11-19T21:40:02,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:02,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:02,542 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:02,546 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:02,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:02,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:02,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:02,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:02,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:02,548 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:02,548 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:02,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-19T21:40:02,551 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:02,552 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:02,552 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:02,553 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,553 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4489f949efd841edb251a3032aa04c49, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a3d196969853431b9eaaf7e86ebb1381, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3459970beb454c7fa24792d489633a5a] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=35.2 K 2024-11-19T21:40:02,554 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,554 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/c537bd317d784ed69e961bcb357b8246, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e7831ae6c590489e8425ca9944c733b9, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8f420f4e75384407a60d727f7ed2194a] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=39.7 K 2024-11-19T21:40:02,556 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 4489f949efd841edb251a3032aa04c49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732052401222 2024-11-19T21:40:02,557 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting c537bd317d784ed69e961bcb357b8246, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732052401174 2024-11-19T21:40:02,558 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a3d196969853431b9eaaf7e86ebb1381, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732052401395 2024-11-19T21:40:02,561 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3459970beb454c7fa24792d489633a5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732052402158 2024-11-19T21:40:02,562 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7831ae6c590489e8425ca9944c733b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732052401395 2024-11-19T21:40:02,566 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f420f4e75384407a60d727f7ed2194a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732052402157 2024-11-19T21:40:02,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e1e426681e9946d8b07a15deb7fbe30d is 50, key is test_row_0/A:col10/1732052402537/Put/seqid=0 2024-11-19T21:40:02,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741846_1022 (size=16681) 2024-11-19T21:40:02,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e1e426681e9946d8b07a15deb7fbe30d 2024-11-19T21:40:02,634 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#10 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:02,635 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/dfe19f71bf8a47bdba3902cb211a6058 is 50, key is test_row_0/B:col10/1732052402166/Put/seqid=0 2024-11-19T21:40:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T21:40:02,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052462608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052462615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/bb38a0f46b2b46babb68fda31a84fe88 is 50, key is test_row_0/B:col10/1732052402537/Put/seqid=0 2024-11-19T21:40:02,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T21:40:02,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:02,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,661 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#12 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:02,662 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/cce459d62c9e4134929944717f578a68 is 50, key is test_row_0/A:col10/1732052402166/Put/seqid=0 2024-11-19T21:40:02,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741847_1023 (size=12104) 2024-11-19T21:40:02,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741848_1024 (size=12001) 2024-11-19T21:40:02,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/bb38a0f46b2b46babb68fda31a84fe88 2024-11-19T21:40:02,696 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/dfe19f71bf8a47bdba3902cb211a6058 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dfe19f71bf8a47bdba3902cb211a6058 2024-11-19T21:40:02,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741849_1025 (size=12104) 2024-11-19T21:40:02,714 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/cce459d62c9e4134929944717f578a68 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/cce459d62c9e4134929944717f578a68 2024-11-19T21:40:02,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/27bc37f68f9544729a5e692834e89afb is 50, key is test_row_0/C:col10/1732052402537/Put/seqid=0 2024-11-19T21:40:02,734 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into cce459d62c9e4134929944717f578a68(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:02,734 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into dfe19f71bf8a47bdba3902cb211a6058(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:02,734 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:02,734 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:02,737 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=13, startTime=1732052402537; duration=0sec 2024-11-19T21:40:02,738 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:02,738 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:02,739 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:02,741 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=13, startTime=1732052402547; duration=0sec 2024-11-19T21:40:02,742 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:02,742 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:02,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741850_1026 (size=12001) 2024-11-19T21:40:02,747 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:02,747 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:02,747 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,747 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d00bd83ba0a42bda1b7ee504d109a28, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/58a01d0e759f4c7f995fd7dd328b63cb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/35a5a06e068e4fb4831cff45715d150c] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=35.2 K 2024-11-19T21:40:02,749 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d00bd83ba0a42bda1b7ee504d109a28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732052401222 2024-11-19T21:40:02,750 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58a01d0e759f4c7f995fd7dd328b63cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732052401395 2024-11-19T21:40:02,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052462740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052462739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,751 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35a5a06e068e4fb4831cff45715d150c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732052402158 2024-11-19T21:40:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052462740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052462749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052462750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,778 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#14 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:02,779 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/c94118691d2b4bf989c195a9074d3da5 is 50, key is test_row_0/C:col10/1732052402166/Put/seqid=0 2024-11-19T21:40:02,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T21:40:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741851_1027 (size=12104) 2024-11-19T21:40:02,841 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/c94118691d2b4bf989c195a9074d3da5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c94118691d2b4bf989c195a9074d3da5 2024-11-19T21:40:02,856 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into c94118691d2b4bf989c195a9074d3da5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:02,856 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:02,857 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=13, startTime=1732052402547; duration=0sec 2024-11-19T21:40:02,857 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:02,857 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T21:40:02,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T21:40:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:02,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:02,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052462959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:02,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052462968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052463061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052463061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052463061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T21:40:03,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:03,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:03,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:03,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:03,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:03,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/27bc37f68f9544729a5e692834e89afb 2024-11-19T21:40:03,150 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T21:40:03,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e1e426681e9946d8b07a15deb7fbe30d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e1e426681e9946d8b07a15deb7fbe30d 2024-11-19T21:40:03,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e1e426681e9946d8b07a15deb7fbe30d, entries=250, sequenceid=78, filesize=16.3 K 2024-11-19T21:40:03,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/bb38a0f46b2b46babb68fda31a84fe88 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/bb38a0f46b2b46babb68fda31a84fe88 2024-11-19T21:40:03,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/bb38a0f46b2b46babb68fda31a84fe88, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T21:40:03,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/27bc37f68f9544729a5e692834e89afb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/27bc37f68f9544729a5e692834e89afb 2024-11-19T21:40:03,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/27bc37f68f9544729a5e692834e89afb, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T21:40:03,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 7f292f8e50ece577fd7cb587fbc66fc6 in 688ms, sequenceid=78, compaction requested=false 2024-11-19T21:40:03,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:03,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T21:40:03,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:03,282 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-19T21:40:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:03,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:03,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/8035087a25434fccac7bd89f2179d053 is 50, key is test_row_0/A:col10/1732052402578/Put/seqid=0 2024-11-19T21:40:03,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741852_1028 (size=9657) 2024-11-19T21:40:03,356 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/8035087a25434fccac7bd89f2179d053 2024-11-19T21:40:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/dd6deb199c1a4ac49803fe4f2d4b7fb9 is 50, key is test_row_0/B:col10/1732052402578/Put/seqid=0 2024-11-19T21:40:03,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741853_1029 (size=9657) 2024-11-19T21:40:03,416 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/dd6deb199c1a4ac49803fe4f2d4b7fb9 2024-11-19T21:40:03,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/dd623138f31a41329bf460f8954743a6 is 50, key is test_row_0/C:col10/1732052402578/Put/seqid=0 2024-11-19T21:40:03,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052463445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052463447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741854_1030 (size=9657) 2024-11-19T21:40:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T21:40:03,460 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/dd623138f31a41329bf460f8954743a6 2024-11-19T21:40:03,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/8035087a25434fccac7bd89f2179d053 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8035087a25434fccac7bd89f2179d053 2024-11-19T21:40:03,488 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8035087a25434fccac7bd89f2179d053, entries=100, sequenceid=91, filesize=9.4 K 2024-11-19T21:40:03,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/dd6deb199c1a4ac49803fe4f2d4b7fb9 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dd6deb199c1a4ac49803fe4f2d4b7fb9 2024-11-19T21:40:03,516 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dd6deb199c1a4ac49803fe4f2d4b7fb9, entries=100, sequenceid=91, filesize=9.4 K 2024-11-19T21:40:03,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/dd623138f31a41329bf460f8954743a6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/dd623138f31a41329bf460f8954743a6 2024-11-19T21:40:03,532 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/dd623138f31a41329bf460f8954743a6, entries=100, sequenceid=91, filesize=9.4 K 2024-11-19T21:40:03,536 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 7f292f8e50ece577fd7cb587fbc66fc6 in 254ms, sequenceid=91, compaction requested=true 2024-11-19T21:40:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:03,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T21:40:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T21:40:03,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T21:40:03,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2010 sec 2024-11-19T21:40:03,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2320 sec 2024-11-19T21:40:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:03,565 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:03,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T21:40:03,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e94b4ee5629c4f4193465904ec35c515 is 50, key is test_row_0/A:col10/1732052403565/Put/seqid=0 2024-11-19T21:40:03,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052463586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052463587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052463590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052463590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052463592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741855_1031 (size=16681) 2024-11-19T21:40:03,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e94b4ee5629c4f4193465904ec35c515 2024-11-19T21:40:03,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/fe5210c3b16a49b0a09c4f070f4995a1 is 50, key is test_row_0/B:col10/1732052403565/Put/seqid=0 2024-11-19T21:40:03,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741856_1032 (size=12001) 2024-11-19T21:40:03,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/fe5210c3b16a49b0a09c4f070f4995a1 2024-11-19T21:40:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052463709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052463710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052463710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052463710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052463710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:03,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:03,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3a7914cdff454831883150258d0545b2 is 50, key is test_row_0/C:col10/1732052403565/Put/seqid=0 2024-11-19T21:40:03,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741857_1033 (size=12001) 2024-11-19T21:40:03,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3a7914cdff454831883150258d0545b2 2024-11-19T21:40:03,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e94b4ee5629c4f4193465904ec35c515 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e94b4ee5629c4f4193465904ec35c515 2024-11-19T21:40:03,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e94b4ee5629c4f4193465904ec35c515, entries=250, sequenceid=119, filesize=16.3 K 2024-11-19T21:40:03,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/fe5210c3b16a49b0a09c4f070f4995a1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fe5210c3b16a49b0a09c4f070f4995a1 2024-11-19T21:40:03,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fe5210c3b16a49b0a09c4f070f4995a1, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T21:40:03,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3a7914cdff454831883150258d0545b2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3a7914cdff454831883150258d0545b2 2024-11-19T21:40:03,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3a7914cdff454831883150258d0545b2, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T21:40:03,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 7f292f8e50ece577fd7cb587fbc66fc6 in 266ms, sequenceid=119, compaction requested=true 2024-11-19T21:40:03,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:03,832 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:03,832 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:03,834 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55123 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:03,835 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:03,835 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:03,835 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/cce459d62c9e4134929944717f578a68, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e1e426681e9946d8b07a15deb7fbe30d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8035087a25434fccac7bd89f2179d053, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e94b4ee5629c4f4193465904ec35c515] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=53.8 K 2024-11-19T21:40:03,836 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:03,836 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:03,836 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:03,837 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dfe19f71bf8a47bdba3902cb211a6058, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/bb38a0f46b2b46babb68fda31a84fe88, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dd6deb199c1a4ac49803fe4f2d4b7fb9, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fe5210c3b16a49b0a09c4f070f4995a1] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=44.7 K 2024-11-19T21:40:03,837 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting cce459d62c9e4134929944717f578a68, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732052402158 2024-11-19T21:40:03,838 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe19f71bf8a47bdba3902cb211a6058, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732052402158 2024-11-19T21:40:03,839 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1e426681e9946d8b07a15deb7fbe30d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732052402382 2024-11-19T21:40:03,839 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting bb38a0f46b2b46babb68fda31a84fe88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732052402401 2024-11-19T21:40:03,840 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8035087a25434fccac7bd89f2179d053, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732052402578 2024-11-19T21:40:03,840 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting dd6deb199c1a4ac49803fe4f2d4b7fb9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732052402578 2024-11-19T21:40:03,841 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting e94b4ee5629c4f4193465904ec35c515, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732052403410 2024-11-19T21:40:03,841 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting fe5210c3b16a49b0a09c4f070f4995a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732052403436 2024-11-19T21:40:03,885 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:03,886 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/9571f98a5fc2468584d1bb2e03acbb1a is 50, key is test_row_0/A:col10/1732052403565/Put/seqid=0 2024-11-19T21:40:03,894 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:03,895 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/dbe9a443a82a4117ac44900732ca6f5a is 50, key is test_row_0/B:col10/1732052403565/Put/seqid=0 2024-11-19T21:40:03,909 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T21:40:03,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741858_1034 (size=12241) 2024-11-19T21:40:03,936 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/9571f98a5fc2468584d1bb2e03acbb1a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9571f98a5fc2468584d1bb2e03acbb1a 2024-11-19T21:40:03,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741859_1035 (size=12241) 2024-11-19T21:40:03,949 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:03,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:40:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:03,959 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into 9571f98a5fc2468584d1bb2e03acbb1a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:03,959 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:03,959 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=12, startTime=1732052403832; duration=0sec 2024-11-19T21:40:03,959 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:03,959 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:03,960 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:03,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/9726383dc7a7418f95bd1d540b11b1bd is 50, key is test_row_0/A:col10/1732052403944/Put/seqid=0 2024-11-19T21:40:03,963 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:03,963 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:03,963 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:03,964 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/dbe9a443a82a4117ac44900732ca6f5a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dbe9a443a82a4117ac44900732ca6f5a 2024-11-19T21:40:03,965 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c94118691d2b4bf989c195a9074d3da5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/27bc37f68f9544729a5e692834e89afb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/dd623138f31a41329bf460f8954743a6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3a7914cdff454831883150258d0545b2] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=44.7 K 2024-11-19T21:40:03,966 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94118691d2b4bf989c195a9074d3da5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732052402158 2024-11-19T21:40:03,967 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27bc37f68f9544729a5e692834e89afb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732052402401 2024-11-19T21:40:03,968 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd623138f31a41329bf460f8954743a6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732052402578 2024-11-19T21:40:03,970 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a7914cdff454831883150258d0545b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732052403436 2024-11-19T21:40:03,980 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into dbe9a443a82a4117ac44900732ca6f5a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:03,980 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:03,980 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=12, startTime=1732052403832; duration=0sec 2024-11-19T21:40:03,981 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:03,981 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741860_1036 (size=14491) 2024-11-19T21:40:04,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/9726383dc7a7418f95bd1d540b11b1bd 2024-11-19T21:40:04,008 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:04,009 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/855f1bee173c4dbe8e3a584a46604ee3 is 50, key is test_row_0/C:col10/1732052403565/Put/seqid=0 2024-11-19T21:40:04,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/9770fc8d584d49069cf20ba08cfe9f15 is 50, key is test_row_0/B:col10/1732052403944/Put/seqid=0 2024-11-19T21:40:04,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052464042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052464043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052464046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052464052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052464054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741861_1037 (size=12241) 2024-11-19T21:40:04,078 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T21:40:04,079 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T21:40:04,081 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/855f1bee173c4dbe8e3a584a46604ee3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/855f1bee173c4dbe8e3a584a46604ee3 2024-11-19T21:40:04,082 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T21:40:04,082 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T21:40:04,083 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T21:40:04,083 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T21:40:04,084 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T21:40:04,084 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T21:40:04,097 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into 855f1bee173c4dbe8e3a584a46604ee3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:04,097 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:04,097 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=12, startTime=1732052403832; duration=0sec 2024-11-19T21:40:04,098 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:04,098 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:04,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741862_1038 (size=12101) 2024-11-19T21:40:04,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052464158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052464169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052464169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052464169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052464169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052464368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052464380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052464380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052464381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052464380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T21:40:04,469 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:04,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T21:40:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T21:40:04,474 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:04,475 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:04,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:04,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/9770fc8d584d49069cf20ba08cfe9f15 2024-11-19T21:40:04,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/016a14574d0f4bf088eb75558beb252b is 50, key is test_row_0/C:col10/1732052403944/Put/seqid=0 2024-11-19T21:40:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T21:40:04,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741863_1039 (size=12101) 2024-11-19T21:40:04,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/016a14574d0f4bf088eb75558beb252b 2024-11-19T21:40:04,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/9726383dc7a7418f95bd1d540b11b1bd as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9726383dc7a7418f95bd1d540b11b1bd 2024-11-19T21:40:04,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9726383dc7a7418f95bd1d540b11b1bd, entries=200, sequenceid=131, filesize=14.2 K 2024-11-19T21:40:04,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/9770fc8d584d49069cf20ba08cfe9f15 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/9770fc8d584d49069cf20ba08cfe9f15 2024-11-19T21:40:04,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:04,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:04,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:04,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:04,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/9770fc8d584d49069cf20ba08cfe9f15, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T21:40:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:04,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/016a14574d0f4bf088eb75558beb252b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/016a14574d0f4bf088eb75558beb252b 2024-11-19T21:40:04,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/016a14574d0f4bf088eb75558beb252b, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T21:40:04,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7f292f8e50ece577fd7cb587fbc66fc6 in 710ms, sequenceid=131, compaction requested=false 2024-11-19T21:40:04,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:04,685 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:04,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:40:04,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/b5e99d9ceca24d7382af2d04d8780452 is 50, key is test_row_0/A:col10/1732052404045/Put/seqid=0 2024-11-19T21:40:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052464723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052464731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052464733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052464737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052464739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741864_1040 (size=14541) 2024-11-19T21:40:04,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/b5e99d9ceca24d7382af2d04d8780452 2024-11-19T21:40:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T21:40:04,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:04,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:04,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:04,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/1a1bf19c84c644cf8c3900e34e9879a3 is 50, key is test_row_0/B:col10/1732052404045/Put/seqid=0 2024-11-19T21:40:04,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741865_1041 (size=12151) 2024-11-19T21:40:04,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/1a1bf19c84c644cf8c3900e34e9879a3 2024-11-19T21:40:04,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052464855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052464855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052464858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052464855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052464856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:04,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/c0da3e9e757b431e971e8de366c932aa is 50, key is test_row_0/C:col10/1732052404045/Put/seqid=0 2024-11-19T21:40:04,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741866_1042 (size=12151) 2024-11-19T21:40:04,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/c0da3e9e757b431e971e8de366c932aa 2024-11-19T21:40:04,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/b5e99d9ceca24d7382af2d04d8780452 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b5e99d9ceca24d7382af2d04d8780452 2024-11-19T21:40:04,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:04,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b5e99d9ceca24d7382af2d04d8780452, entries=200, sequenceid=159, filesize=14.2 K 2024-11-19T21:40:04,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:04,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/1a1bf19c84c644cf8c3900e34e9879a3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1a1bf19c84c644cf8c3900e34e9879a3 2024-11-19T21:40:04,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1a1bf19c84c644cf8c3900e34e9879a3, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T21:40:04,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/c0da3e9e757b431e971e8de366c932aa as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c0da3e9e757b431e971e8de366c932aa 2024-11-19T21:40:05,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c0da3e9e757b431e971e8de366c932aa, entries=150, sequenceid=159, filesize=11.9 K 2024-11-19T21:40:05,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 7f292f8e50ece577fd7cb587fbc66fc6 in 322ms, sequenceid=159, compaction requested=true 2024-11-19T21:40:05,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:05,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:05,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:05,008 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:05,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:05,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:05,009 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:05,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:05,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:05,011 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:05,011 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:05,011 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,012 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dbe9a443a82a4117ac44900732ca6f5a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/9770fc8d584d49069cf20ba08cfe9f15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1a1bf19c84c644cf8c3900e34e9879a3] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=35.6 K 2024-11-19T21:40:05,012 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41273 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:05,012 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:05,013 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,013 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9571f98a5fc2468584d1bb2e03acbb1a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9726383dc7a7418f95bd1d540b11b1bd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b5e99d9ceca24d7382af2d04d8780452] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=40.3 K 2024-11-19T21:40:05,014 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9571f98a5fc2468584d1bb2e03acbb1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732052403436 2024-11-19T21:40:05,015 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting dbe9a443a82a4117ac44900732ca6f5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732052403436 2024-11-19T21:40:05,015 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9726383dc7a7418f95bd1d540b11b1bd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732052403933 2024-11-19T21:40:05,016 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 9770fc8d584d49069cf20ba08cfe9f15, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732052403933 2024-11-19T21:40:05,016 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5e99d9ceca24d7382af2d04d8780452, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732052404025 2024-11-19T21:40:05,016 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a1bf19c84c644cf8c3900e34e9879a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732052404040 2024-11-19T21:40:05,055 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:05,056 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/67fc3bf860b348c5a899c6e09910711b is 50, key is test_row_0/A:col10/1732052404045/Put/seqid=0 2024-11-19T21:40:05,067 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:05,068 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/6afd95c8aba642e1bc3aba2aa9bc7531 is 50, key is test_row_0/B:col10/1732052404045/Put/seqid=0 2024-11-19T21:40:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:05,078 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:05,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T21:40:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T21:40:05,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:05,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:05,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741867_1043 (size=12493) 2024-11-19T21:40:05,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/fb2f872de0904635882fe94c07ec8294 is 50, key is test_row_0/A:col10/1732052404732/Put/seqid=0 2024-11-19T21:40:05,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741868_1044 (size=12493) 2024-11-19T21:40:05,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741869_1045 (size=14541) 2024-11-19T21:40:05,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:05,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:05,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052465308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052465309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052465314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052465321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052465319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052465429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052465429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052465439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052465448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052465448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:05,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:05,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,539 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/67fc3bf860b348c5a899c6e09910711b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/67fc3bf860b348c5a899c6e09910711b 2024-11-19T21:40:05,553 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into 67fc3bf860b348c5a899c6e09910711b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:05,553 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:05,553 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=13, startTime=1732052405008; duration=0sec 2024-11-19T21:40:05,553 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:05,553 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:05,554 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:05,555 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:05,555 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:05,556 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,556 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/855f1bee173c4dbe8e3a584a46604ee3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/016a14574d0f4bf088eb75558beb252b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c0da3e9e757b431e971e8de366c932aa] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=35.6 K 2024-11-19T21:40:05,563 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 855f1bee173c4dbe8e3a584a46604ee3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732052403436 2024-11-19T21:40:05,565 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 016a14574d0f4bf088eb75558beb252b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732052403933 2024-11-19T21:40:05,568 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0da3e9e757b431e971e8de366c932aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732052404040 2024-11-19T21:40:05,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/fb2f872de0904635882fe94c07ec8294 2024-11-19T21:40:05,578 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/6afd95c8aba642e1bc3aba2aa9bc7531 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6afd95c8aba642e1bc3aba2aa9bc7531 2024-11-19T21:40:05,589 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:05,590 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/2bb85fd1c6f74de38a59c655e7a898ed is 50, key is test_row_0/C:col10/1732052404045/Put/seqid=0 2024-11-19T21:40:05,596 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into 6afd95c8aba642e1bc3aba2aa9bc7531(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:05,596 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:05,596 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=13, startTime=1732052405008; duration=0sec 2024-11-19T21:40:05,596 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:05,596 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:05,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/85eca0a4f11b42db94556e2ee337c450 is 50, key is test_row_0/B:col10/1732052404732/Put/seqid=0 2024-11-19T21:40:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T21:40:05,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741870_1046 (size=12493) 2024-11-19T21:40:05,647 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/2bb85fd1c6f74de38a59c655e7a898ed as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2bb85fd1c6f74de38a59c655e7a898ed 2024-11-19T21:40:05,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052465648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052465648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,659 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into 2bb85fd1c6f74de38a59c655e7a898ed(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:05,659 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:05,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,659 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=13, startTime=1732052405009; duration=0sec 2024-11-19T21:40:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052465648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,659 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:05,659 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052465659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052465659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741871_1047 (size=12151) 2024-11-19T21:40:05,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:05,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:05,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:05,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:05,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:05,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052465959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052465970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052465971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052465979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052465979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/85eca0a4f11b42db94556e2ee337c450 2024-11-19T21:40:06,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:06,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:06,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/8f8315e879654bc3bae560b07b404243 is 50, key is test_row_0/C:col10/1732052404732/Put/seqid=0 2024-11-19T21:40:06,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741872_1048 (size=12151) 2024-11-19T21:40:06,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/8f8315e879654bc3bae560b07b404243 2024-11-19T21:40:06,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/fb2f872de0904635882fe94c07ec8294 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fb2f872de0904635882fe94c07ec8294 2024-11-19T21:40:06,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fb2f872de0904635882fe94c07ec8294, entries=200, sequenceid=173, filesize=14.2 K 2024-11-19T21:40:06,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/85eca0a4f11b42db94556e2ee337c450 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/85eca0a4f11b42db94556e2ee337c450 2024-11-19T21:40:06,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/85eca0a4f11b42db94556e2ee337c450, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T21:40:06,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/8f8315e879654bc3bae560b07b404243 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8f8315e879654bc3bae560b07b404243 2024-11-19T21:40:06,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8f8315e879654bc3bae560b07b404243, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T21:40:06,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 7f292f8e50ece577fd7cb587fbc66fc6 in 1129ms, sequenceid=173, compaction requested=false 2024-11-19T21:40:06,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:06,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T21:40:06,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,245 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:40:06,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/ef6925d2a67c4fc58a1c345e634684bc is 50, key is test_row_0/A:col10/1732052405306/Put/seqid=0 2024-11-19T21:40:06,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741873_1049 (size=12151) 2024-11-19T21:40:06,332 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/ef6925d2a67c4fc58a1c345e634684bc 2024-11-19T21:40:06,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/a4407ac222ca44c78b52eb1e3e41d5c0 is 50, key is test_row_0/B:col10/1732052405306/Put/seqid=0 2024-11-19T21:40:06,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741874_1050 (size=12151) 2024-11-19T21:40:06,379 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/a4407ac222ca44c78b52eb1e3e41d5c0 2024-11-19T21:40:06,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/5ed12c66a40a4ee1a2fbcd94e7f0813f is 50, key is test_row_0/C:col10/1732052405306/Put/seqid=0 2024-11-19T21:40:06,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741875_1051 (size=12151) 2024-11-19T21:40:06,425 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/5ed12c66a40a4ee1a2fbcd94e7f0813f 2024-11-19T21:40:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/ef6925d2a67c4fc58a1c345e634684bc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/ef6925d2a67c4fc58a1c345e634684bc 2024-11-19T21:40:06,462 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/ef6925d2a67c4fc58a1c345e634684bc, entries=150, sequenceid=199, filesize=11.9 K 2024-11-19T21:40:06,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/a4407ac222ca44c78b52eb1e3e41d5c0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a4407ac222ca44c78b52eb1e3e41d5c0 2024-11-19T21:40:06,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:06,496 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a4407ac222ca44c78b52eb1e3e41d5c0, entries=150, sequenceid=199, filesize=11.9 K 2024-11-19T21:40:06,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/5ed12c66a40a4ee1a2fbcd94e7f0813f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5ed12c66a40a4ee1a2fbcd94e7f0813f 2024-11-19T21:40:06,510 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5ed12c66a40a4ee1a2fbcd94e7f0813f, entries=150, sequenceid=199, filesize=11.9 K 2024-11-19T21:40:06,514 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=33.54 KB/34350 for 7f292f8e50ece577fd7cb587fbc66fc6 in 269ms, sequenceid=199, compaction requested=true 2024-11-19T21:40:06,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:06,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T21:40:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T21:40:06,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T21:40:06,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0420 sec 2024-11-19T21:40:06,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.0520 sec 2024-11-19T21:40:06,532 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:06,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:40:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:06,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/08eb804efc224da0ab754e789bd0c4e4 is 50, key is test_row_0/A:col10/1732052406530/Put/seqid=0 2024-11-19T21:40:06,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741876_1052 (size=12151) 2024-11-19T21:40:06,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/08eb804efc224da0ab754e789bd0c4e4 2024-11-19T21:40:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T21:40:06,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/d22994a628cc46df94eb05949c12fbb0 is 50, key is test_row_0/B:col10/1732052406530/Put/seqid=0 2024-11-19T21:40:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052466601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,625 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052466605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052466604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052466622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T21:40:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T21:40:06,632 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:06,634 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:06,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:06,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052466606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741877_1053 (size=12151) 2024-11-19T21:40:06,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/d22994a628cc46df94eb05949c12fbb0 2024-11-19T21:40:06,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/fbb590c9fd1d4c3fbea1ecb4b4a16b69 is 50, key is test_row_0/C:col10/1732052406530/Put/seqid=0 2024-11-19T21:40:06,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052466728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T21:40:06,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741878_1054 (size=12151) 2024-11-19T21:40:06,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/fbb590c9fd1d4c3fbea1ecb4b4a16b69 2024-11-19T21:40:06,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052466739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052466739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052466739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/08eb804efc224da0ab754e789bd0c4e4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/08eb804efc224da0ab754e789bd0c4e4 2024-11-19T21:40:06,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052466749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/08eb804efc224da0ab754e789bd0c4e4, entries=150, sequenceid=210, filesize=11.9 K 2024-11-19T21:40:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/d22994a628cc46df94eb05949c12fbb0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/d22994a628cc46df94eb05949c12fbb0 2024-11-19T21:40:06,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/d22994a628cc46df94eb05949c12fbb0, entries=150, sequenceid=210, filesize=11.9 K 2024-11-19T21:40:06,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/fbb590c9fd1d4c3fbea1ecb4b4a16b69 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/fbb590c9fd1d4c3fbea1ecb4b4a16b69 2024-11-19T21:40:06,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T21:40:06,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:06,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:06,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:06,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/fbb590c9fd1d4c3fbea1ecb4b4a16b69, entries=150, sequenceid=210, filesize=11.9 K 2024-11-19T21:40:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:06,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7f292f8e50ece577fd7cb587fbc66fc6 in 261ms, sequenceid=210, compaction requested=true 2024-11-19T21:40:06,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:06,794 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T21:40:06,794 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:06,799 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:06,799 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:06,799 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,799 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/67fc3bf860b348c5a899c6e09910711b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fb2f872de0904635882fe94c07ec8294, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/ef6925d2a67c4fc58a1c345e634684bc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/08eb804efc224da0ab754e789bd0c4e4] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=50.1 K 2024-11-19T21:40:06,800 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:06,800 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:06,800 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67fc3bf860b348c5a899c6e09910711b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732052404040 2024-11-19T21:40:06,800 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,800 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6afd95c8aba642e1bc3aba2aa9bc7531, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/85eca0a4f11b42db94556e2ee337c450, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a4407ac222ca44c78b52eb1e3e41d5c0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/d22994a628cc46df94eb05949c12fbb0] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=47.8 K 2024-11-19T21:40:06,802 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6afd95c8aba642e1bc3aba2aa9bc7531, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732052404040 2024-11-19T21:40:06,802 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb2f872de0904635882fe94c07ec8294, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732052404722 2024-11-19T21:40:06,802 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 85eca0a4f11b42db94556e2ee337c450, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732052404732 2024-11-19T21:40:06,803 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6925d2a67c4fc58a1c345e634684bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732052405263 2024-11-19T21:40:06,803 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a4407ac222ca44c78b52eb1e3e41d5c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732052405263 2024-11-19T21:40:06,803 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08eb804efc224da0ab754e789bd0c4e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732052406479 2024-11-19T21:40:06,804 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d22994a628cc46df94eb05949c12fbb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732052406479 2024-11-19T21:40:06,845 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:06,846 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/a1394b1783b64ff6a5773502fb2b1cef is 50, key is test_row_0/A:col10/1732052406530/Put/seqid=0 2024-11-19T21:40:06,861 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:06,862 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/8f6b74996cab4ab4bc9386f5936cc1e3 is 50, key is test_row_0/B:col10/1732052406530/Put/seqid=0 2024-11-19T21:40:06,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741880_1056 (size=12629) 2024-11-19T21:40:06,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741879_1055 (size=12629) 2024-11-19T21:40:06,922 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/8f6b74996cab4ab4bc9386f5936cc1e3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/8f6b74996cab4ab4bc9386f5936cc1e3 2024-11-19T21:40:06,932 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/a1394b1783b64ff6a5773502fb2b1cef as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/a1394b1783b64ff6a5773502fb2b1cef 2024-11-19T21:40:06,933 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into 8f6b74996cab4ab4bc9386f5936cc1e3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:06,933 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:06,933 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=12, startTime=1732052406794; duration=0sec 2024-11-19T21:40:06,933 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:06,934 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:06,934 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:06,936 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:06,936 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:06,937 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,937 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2bb85fd1c6f74de38a59c655e7a898ed, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8f8315e879654bc3bae560b07b404243, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5ed12c66a40a4ee1a2fbcd94e7f0813f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/fbb590c9fd1d4c3fbea1ecb4b4a16b69] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=47.8 K 2024-11-19T21:40:06,938 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bb85fd1c6f74de38a59c655e7a898ed, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732052404040 2024-11-19T21:40:06,938 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f8315e879654bc3bae560b07b404243, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732052404732 2024-11-19T21:40:06,939 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed12c66a40a4ee1a2fbcd94e7f0813f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732052405263 2024-11-19T21:40:06,940 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb590c9fd1d4c3fbea1ecb4b4a16b69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732052406479 2024-11-19T21:40:06,944 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into a1394b1783b64ff6a5773502fb2b1cef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:06,944 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:06,944 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=12, startTime=1732052406794; duration=0sec 2024-11-19T21:40:06,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T21:40:06,944 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:06,944 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:06,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:06,945 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:40:06,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T21:40:06,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/91178f42241e448b856c0f2e5b5644f9 is 50, key is test_row_0/A:col10/1732052406943/Put/seqid=0 2024-11-19T21:40:06,959 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:06,959 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/cd6fca96feae4347b18a3410398c3521 is 50, key is test_row_0/C:col10/1732052406530/Put/seqid=0 2024-11-19T21:40:06,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052466966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052466967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052466967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052466968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052466974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:06,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741882_1058 (size=12629) 2024-11-19T21:40:06,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741881_1057 (size=14541) 2024-11-19T21:40:06,991 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/91178f42241e448b856c0f2e5b5644f9 2024-11-19T21:40:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/4353d20206d54fb8a736404967ce3a1a is 50, key is test_row_0/B:col10/1732052406943/Put/seqid=0 2024-11-19T21:40:07,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741883_1059 (size=12151) 2024-11-19T21:40:07,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052467078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052467078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052467078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052467089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T21:40:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052467288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052467306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052467306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052467308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052467306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,404 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/cd6fca96feae4347b18a3410398c3521 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/cd6fca96feae4347b18a3410398c3521 2024-11-19T21:40:07,423 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into cd6fca96feae4347b18a3410398c3521(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:07,423 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:07,423 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=12, startTime=1732052406794; duration=0sec 2024-11-19T21:40:07,423 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:07,424 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:07,452 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/4353d20206d54fb8a736404967ce3a1a 2024-11-19T21:40:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/0e970455ea8a4f47b7d05b4ba79fdb66 is 50, key is test_row_0/C:col10/1732052406943/Put/seqid=0 2024-11-19T21:40:07,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741884_1060 (size=12151) 2024-11-19T21:40:07,516 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/0e970455ea8a4f47b7d05b4ba79fdb66 2024-11-19T21:40:07,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/91178f42241e448b856c0f2e5b5644f9 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/91178f42241e448b856c0f2e5b5644f9 2024-11-19T21:40:07,545 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/91178f42241e448b856c0f2e5b5644f9, entries=200, sequenceid=238, filesize=14.2 K 2024-11-19T21:40:07,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/4353d20206d54fb8a736404967ce3a1a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4353d20206d54fb8a736404967ce3a1a 2024-11-19T21:40:07,562 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4353d20206d54fb8a736404967ce3a1a, entries=150, sequenceid=238, filesize=11.9 K 2024-11-19T21:40:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/0e970455ea8a4f47b7d05b4ba79fdb66 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0e970455ea8a4f47b7d05b4ba79fdb66 2024-11-19T21:40:07,579 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0e970455ea8a4f47b7d05b4ba79fdb66, entries=150, sequenceid=238, filesize=11.9 K 2024-11-19T21:40:07,583 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7f292f8e50ece577fd7cb587fbc66fc6 in 639ms, sequenceid=238, compaction requested=false 2024-11-19T21:40:07,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:07,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:07,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T21:40:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T21:40:07,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T21:40:07,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 952 msec 2024-11-19T21:40:07,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 965 msec 2024-11-19T21:40:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:07,624 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:07,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:40:07,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/58bf11fb1af945f782b55e13c18647c2 is 50, key is test_row_0/A:col10/1732052406961/Put/seqid=0 2024-11-19T21:40:07,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741885_1061 (size=14541) 2024-11-19T21:40:07,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/58bf11fb1af945f782b55e13c18647c2 2024-11-19T21:40:07,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052467702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/21f44f787e0348b4bb8b4bba1f8541b0 is 50, key is test_row_0/B:col10/1732052406961/Put/seqid=0 2024-11-19T21:40:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052467704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052467708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052467708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741886_1062 (size=12151) 2024-11-19T21:40:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T21:40:07,769 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:07,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T21:40:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T21:40:07,774 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:07,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:07,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:07,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052467799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052467818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052467819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052467820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052467819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T21:40:07,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T21:40:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:07,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052468028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052468029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052468030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052468038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T21:40:08,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T21:40:08,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:08,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/21f44f787e0348b4bb8b4bba1f8541b0 2024-11-19T21:40:08,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/8040697b878c488fb2f3e1e3ec935178 is 50, key is test_row_0/C:col10/1732052406961/Put/seqid=0 2024-11-19T21:40:08,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741887_1063 (size=12151) 2024-11-19T21:40:08,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T21:40:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052468341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052468342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052468348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052468358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T21:40:08,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T21:40:08,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:08,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T21:40:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:08,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/8040697b878c488fb2f3e1e3ec935178 2024-11-19T21:40:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/58bf11fb1af945f782b55e13c18647c2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/58bf11fb1af945f782b55e13c18647c2 2024-11-19T21:40:08,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/58bf11fb1af945f782b55e13c18647c2, entries=200, sequenceid=250, filesize=14.2 K 2024-11-19T21:40:08,619 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T21:40:08,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/21f44f787e0348b4bb8b4bba1f8541b0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/21f44f787e0348b4bb8b4bba1f8541b0 2024-11-19T21:40:08,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/21f44f787e0348b4bb8b4bba1f8541b0, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T21:40:08,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/8040697b878c488fb2f3e1e3ec935178 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8040697b878c488fb2f3e1e3ec935178 2024-11-19T21:40:08,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8040697b878c488fb2f3e1e3ec935178, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T21:40:08,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7f292f8e50ece577fd7cb587fbc66fc6 in 1034ms, sequenceid=250, compaction requested=true 2024-11-19T21:40:08,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:08,659 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:08,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:08,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:08,659 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:08,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:08,661 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:08,661 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:08,661 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,661 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/a1394b1783b64ff6a5773502fb2b1cef, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/91178f42241e448b856c0f2e5b5644f9, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/58bf11fb1af945f782b55e13c18647c2] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=40.7 K 2024-11-19T21:40:08,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:08,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:08,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:08,662 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:08,663 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:08,663 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,663 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/8f6b74996cab4ab4bc9386f5936cc1e3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4353d20206d54fb8a736404967ce3a1a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/21f44f787e0348b4bb8b4bba1f8541b0] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=36.1 K 2024-11-19T21:40:08,664 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1394b1783b64ff6a5773502fb2b1cef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732052406479 2024-11-19T21:40:08,664 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f6b74996cab4ab4bc9386f5936cc1e3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732052406479 2024-11-19T21:40:08,664 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91178f42241e448b856c0f2e5b5644f9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732052406588 2024-11-19T21:40:08,665 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 4353d20206d54fb8a736404967ce3a1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732052406588 2024-11-19T21:40:08,665 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58bf11fb1af945f782b55e13c18647c2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732052406961 2024-11-19T21:40:08,666 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 21f44f787e0348b4bb8b4bba1f8541b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732052406961 2024-11-19T21:40:08,689 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:08,690 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/0988c363d28d4f3aa3f477502b2b7053 is 50, key is test_row_0/A:col10/1732052406961/Put/seqid=0 2024-11-19T21:40:08,698 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:08,699 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/66adef52b81d4ce1ab6bf8c60953aafb is 50, key is test_row_0/B:col10/1732052406961/Put/seqid=0 2024-11-19T21:40:08,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T21:40:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,732 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:40:08,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741888_1064 (size=12731) 2024-11-19T21:40:08,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741889_1065 (size=12731) 2024-11-19T21:40:08,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/123e319ccc2843c1bdf2a814aa8f1907 is 50, key is test_row_0/A:col10/1732052407704/Put/seqid=0 2024-11-19T21:40:08,767 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/66adef52b81d4ce1ab6bf8c60953aafb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/66adef52b81d4ce1ab6bf8c60953aafb 2024-11-19T21:40:08,786 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into 66adef52b81d4ce1ab6bf8c60953aafb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:08,787 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:08,787 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=13, startTime=1732052408659; duration=0sec 2024-11-19T21:40:08,787 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:08,787 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:08,787 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:08,789 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:08,789 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:08,789 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:08,790 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/cd6fca96feae4347b18a3410398c3521, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0e970455ea8a4f47b7d05b4ba79fdb66, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8040697b878c488fb2f3e1e3ec935178] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=36.1 K 2024-11-19T21:40:08,791 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting cd6fca96feae4347b18a3410398c3521, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732052406479 2024-11-19T21:40:08,792 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e970455ea8a4f47b7d05b4ba79fdb66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732052406588 2024-11-19T21:40:08,792 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8040697b878c488fb2f3e1e3ec935178, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732052406961 2024-11-19T21:40:08,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741890_1066 (size=12301) 2024-11-19T21:40:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:08,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:08,837 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:08,838 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/8267a6a069144b20bf370f298fad5ad3 is 50, key is test_row_0/C:col10/1732052406961/Put/seqid=0 2024-11-19T21:40:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052468855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052468859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052468865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052468869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052468878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741891_1067 (size=12731) 2024-11-19T21:40:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T21:40:08,919 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/8267a6a069144b20bf370f298fad5ad3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8267a6a069144b20bf370f298fad5ad3 2024-11-19T21:40:08,931 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into 8267a6a069144b20bf370f298fad5ad3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:08,931 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:08,931 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=13, startTime=1732052408661; duration=0sec 2024-11-19T21:40:08,931 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:08,931 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:08,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052468969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052468969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052468978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,164 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/0988c363d28d4f3aa3f477502b2b7053 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/0988c363d28d4f3aa3f477502b2b7053 2024-11-19T21:40:09,176 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into 0988c363d28d4f3aa3f477502b2b7053(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:09,176 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:09,177 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=13, startTime=1732052408659; duration=0sec 2024-11-19T21:40:09,177 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:09,177 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:09,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052469200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052469201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052469202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,214 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/123e319ccc2843c1bdf2a814aa8f1907 2024-11-19T21:40:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/aea1d28835114915af2c0d176239ec51 is 50, key is test_row_0/B:col10/1732052407704/Put/seqid=0 2024-11-19T21:40:09,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741892_1068 (size=12301) 2024-11-19T21:40:09,281 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/aea1d28835114915af2c0d176239ec51 2024-11-19T21:40:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/e59b1126470043f89f3a0560ae594342 is 50, key is test_row_0/C:col10/1732052407704/Put/seqid=0 2024-11-19T21:40:09,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741893_1069 (size=12301) 2024-11-19T21:40:09,330 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/e59b1126470043f89f3a0560ae594342 2024-11-19T21:40:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/123e319ccc2843c1bdf2a814aa8f1907 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/123e319ccc2843c1bdf2a814aa8f1907 2024-11-19T21:40:09,357 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/123e319ccc2843c1bdf2a814aa8f1907, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T21:40:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/aea1d28835114915af2c0d176239ec51 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/aea1d28835114915af2c0d176239ec51 2024-11-19T21:40:09,373 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/aea1d28835114915af2c0d176239ec51, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T21:40:09,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/e59b1126470043f89f3a0560ae594342 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/e59b1126470043f89f3a0560ae594342 2024-11-19T21:40:09,390 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/e59b1126470043f89f3a0560ae594342, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T21:40:09,392 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7f292f8e50ece577fd7cb587fbc66fc6 in 659ms, sequenceid=275, compaction requested=false 2024-11-19T21:40:09,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:09,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:09,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T21:40:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T21:40:09,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T21:40:09,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6210 sec 2024-11-19T21:40:09,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6310 sec 2024-11-19T21:40:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:09,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:40:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:09,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/56ccf5abcfba461da0eb83bf29fc4dac is 50, key is test_row_0/A:col10/1732052408850/Put/seqid=0 2024-11-19T21:40:09,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741894_1070 (size=14741) 2024-11-19T21:40:09,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/56ccf5abcfba461da0eb83bf29fc4dac 2024-11-19T21:40:09,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/0453ab769c694ab397542fee30a8351a is 50, key is test_row_0/B:col10/1732052408850/Put/seqid=0 2024-11-19T21:40:09,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052469584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052469585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052469586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741895_1071 (size=12301) 2024-11-19T21:40:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052469698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052469698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052469698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052469888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,896 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in 7f292f8e50ece577fd7cb587fbc66fc6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052469898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,901 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 7f292f8e50ece577fd7cb587fbc66fc6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052469909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052469909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052469909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T21:40:09,919 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:09,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T21:40:09,921 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T21:40:09,922 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:09,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:09,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/0453ab769c694ab397542fee30a8351a 2024-11-19T21:40:10,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3eff43391c3547889adf4f3da616afb3 is 50, key is test_row_0/C:col10/1732052408850/Put/seqid=0 2024-11-19T21:40:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T21:40:10,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741896_1072 (size=12301) 2024-11-19T21:40:10,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3eff43391c3547889adf4f3da616afb3 2024-11-19T21:40:10,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/56ccf5abcfba461da0eb83bf29fc4dac as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/56ccf5abcfba461da0eb83bf29fc4dac 2024-11-19T21:40:10,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/56ccf5abcfba461da0eb83bf29fc4dac, entries=200, sequenceid=290, filesize=14.4 K 2024-11-19T21:40:10,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/0453ab769c694ab397542fee30a8351a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0453ab769c694ab397542fee30a8351a 2024-11-19T21:40:10,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0453ab769c694ab397542fee30a8351a, entries=150, sequenceid=290, filesize=12.0 K 2024-11-19T21:40:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3eff43391c3547889adf4f3da616afb3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3eff43391c3547889adf4f3da616afb3 2024-11-19T21:40:10,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3eff43391c3547889adf4f3da616afb3, entries=150, sequenceid=290, filesize=12.0 K 2024-11-19T21:40:10,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T21:40:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 7f292f8e50ece577fd7cb587fbc66fc6 in 560ms, sequenceid=290, compaction requested=true 2024-11-19T21:40:10,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:10,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,078 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:10,078 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:10,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:10,079 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:10,079 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:10,079 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,080 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/0988c363d28d4f3aa3f477502b2b7053, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/123e319ccc2843c1bdf2a814aa8f1907, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/56ccf5abcfba461da0eb83bf29fc4dac] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=38.8 K 2024-11-19T21:40:10,081 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:10,081 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:10,081 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,081 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/66adef52b81d4ce1ab6bf8c60953aafb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/aea1d28835114915af2c0d176239ec51, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0453ab769c694ab397542fee30a8351a] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=36.5 K 2024-11-19T21:40:10,082 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0988c363d28d4f3aa3f477502b2b7053, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732052406961 2024-11-19T21:40:10,083 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 66adef52b81d4ce1ab6bf8c60953aafb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732052406961 2024-11-19T21:40:10,083 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 123e319ccc2843c1bdf2a814aa8f1907, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732052407694 2024-11-19T21:40:10,083 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting aea1d28835114915af2c0d176239ec51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732052407694 2024-11-19T21:40:10,084 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56ccf5abcfba461da0eb83bf29fc4dac, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732052408850 2024-11-19T21:40:10,084 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0453ab769c694ab397542fee30a8351a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732052408850 2024-11-19T21:40:10,100 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:10,101 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/4995767f532c4c47b302a7a3d225760a is 50, key is test_row_0/B:col10/1732052408850/Put/seqid=0 2024-11-19T21:40:10,123 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#61 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:10,124 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/d0e59a4d5f8042fe99223aba6d3d2d70 is 50, key is test_row_0/A:col10/1732052408850/Put/seqid=0 2024-11-19T21:40:10,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741897_1073 (size=12983) 2024-11-19T21:40:10,152 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/4995767f532c4c47b302a7a3d225760a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4995767f532c4c47b302a7a3d225760a 2024-11-19T21:40:10,166 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into 4995767f532c4c47b302a7a3d225760a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:10,166 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:10,166 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=13, startTime=1732052410078; duration=0sec 2024-11-19T21:40:10,167 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:10,167 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:10,167 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:10,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741898_1074 (size=12983) 2024-11-19T21:40:10,169 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:10,169 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:10,169 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,170 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8267a6a069144b20bf370f298fad5ad3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/e59b1126470043f89f3a0560ae594342, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3eff43391c3547889adf4f3da616afb3] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=36.5 K 2024-11-19T21:40:10,170 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8267a6a069144b20bf370f298fad5ad3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732052406961 2024-11-19T21:40:10,171 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e59b1126470043f89f3a0560ae594342, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732052407694 2024-11-19T21:40:10,172 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eff43391c3547889adf4f3da616afb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732052408850 2024-11-19T21:40:10,178 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/d0e59a4d5f8042fe99223aba6d3d2d70 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d0e59a4d5f8042fe99223aba6d3d2d70 2024-11-19T21:40:10,186 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:10,187 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/50c8e9ab17a643e3addd573163928bda is 50, key is test_row_0/C:col10/1732052408850/Put/seqid=0 2024-11-19T21:40:10,190 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into d0e59a4d5f8042fe99223aba6d3d2d70(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:10,191 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:10,191 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=13, startTime=1732052410077; duration=0sec 2024-11-19T21:40:10,192 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:10,192 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:10,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741899_1075 (size=12983) 2024-11-19T21:40:10,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:10,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T21:40:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:10,230 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/50c8e9ab17a643e3addd573163928bda as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/50c8e9ab17a643e3addd573163928bda 2024-11-19T21:40:10,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T21:40:10,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/1587a5b29ece4d08af46cb80cab99f39 is 50, key is test_row_0/A:col10/1732052409583/Put/seqid=0 2024-11-19T21:40:10,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:10,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T21:40:10,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052470241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,249 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into 50c8e9ab17a643e3addd573163928bda(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:10,250 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:10,250 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=13, startTime=1732052410079; duration=0sec 2024-11-19T21:40:10,250 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:10,250 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:10,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741900_1076 (size=12301) 2024-11-19T21:40:10,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052470245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052470248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052470358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052470358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052470368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T21:40:10,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T21:40:10,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:10,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T21:40:10,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052470569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052470568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052470578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/1587a5b29ece4d08af46cb80cab99f39 2024-11-19T21:40:10,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/2067da6f6fd846a39f31f33b4953c070 is 50, key is test_row_0/B:col10/1732052409583/Put/seqid=0 2024-11-19T21:40:10,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741901_1077 (size=12301) 2024-11-19T21:40:10,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/2067da6f6fd846a39f31f33b4953c070 2024-11-19T21:40:10,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/da81f3bde3ae4c88b8646786236ae8e0 is 50, key is test_row_0/C:col10/1732052409583/Put/seqid=0 2024-11-19T21:40:10,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T21:40:10,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:10,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741902_1078 (size=12301) 2024-11-19T21:40:10,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T21:40:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:10,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:10,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:10,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052470879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052470879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:10,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052470888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T21:40:11,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:11,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:11,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:11,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:11,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T21:40:11,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/da81f3bde3ae4c88b8646786236ae8e0 2024-11-19T21:40:11,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/1587a5b29ece4d08af46cb80cab99f39 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/1587a5b29ece4d08af46cb80cab99f39 2024-11-19T21:40:11,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/1587a5b29ece4d08af46cb80cab99f39, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T21:40:11,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/2067da6f6fd846a39f31f33b4953c070 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2067da6f6fd846a39f31f33b4953c070 2024-11-19T21:40:11,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2067da6f6fd846a39f31f33b4953c070, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T21:40:11,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/da81f3bde3ae4c88b8646786236ae8e0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/da81f3bde3ae4c88b8646786236ae8e0 2024-11-19T21:40:11,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T21:40:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:11,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:11,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/da81f3bde3ae4c88b8646786236ae8e0, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T21:40:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:11,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 7f292f8e50ece577fd7cb587fbc66fc6 in 964ms, sequenceid=319, compaction requested=false 2024-11-19T21:40:11,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:11,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T21:40:11,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:11,337 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-19T21:40:11,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/8e9478d7dc6f49deb5aa6de9cf80c60e is 50, key is test_row_0/A:col10/1732052410239/Put/seqid=0 2024-11-19T21:40:11,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741903_1079 (size=12301) 2024-11-19T21:40:11,378 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/8e9478d7dc6f49deb5aa6de9cf80c60e 2024-11-19T21:40:11,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/3a5f27e7c09c4bd3bf8f605bb69f0237 is 50, key is test_row_0/B:col10/1732052410239/Put/seqid=0 2024-11-19T21:40:11,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:11,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741904_1080 (size=12301) 2024-11-19T21:40:11,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052471478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052471479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052471480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052471588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052471588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052471588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052471798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052471798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052471798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,837 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/3a5f27e7c09c4bd3bf8f605bb69f0237 2024-11-19T21:40:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/bad9a8e72c1a49c7a0425a6f5a3d04e3 is 50, key is test_row_0/C:col10/1732052410239/Put/seqid=0 2024-11-19T21:40:11,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741905_1081 (size=12301) 2024-11-19T21:40:11,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052471919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052471919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:11,927 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 7f292f8e50ece577fd7cb587fbc66fc6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:11,928 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in 7f292f8e50ece577fd7cb587fbc66fc6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T21:40:12,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052472108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052472108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052472109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,295 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/bad9a8e72c1a49c7a0425a6f5a3d04e3 2024-11-19T21:40:12,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/8e9478d7dc6f49deb5aa6de9cf80c60e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8e9478d7dc6f49deb5aa6de9cf80c60e 2024-11-19T21:40:12,314 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8e9478d7dc6f49deb5aa6de9cf80c60e, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T21:40:12,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/3a5f27e7c09c4bd3bf8f605bb69f0237 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3a5f27e7c09c4bd3bf8f605bb69f0237 2024-11-19T21:40:12,324 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3a5f27e7c09c4bd3bf8f605bb69f0237, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T21:40:12,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/bad9a8e72c1a49c7a0425a6f5a3d04e3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bad9a8e72c1a49c7a0425a6f5a3d04e3 2024-11-19T21:40:12,344 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bad9a8e72c1a49c7a0425a6f5a3d04e3, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T21:40:12,352 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 7f292f8e50ece577fd7cb587fbc66fc6 in 1015ms, sequenceid=329, compaction requested=true 2024-11-19T21:40:12,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:12,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:12,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T21:40:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T21:40:12,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T21:40:12,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4310 sec 2024-11-19T21:40:12,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.4390 sec 2024-11-19T21:40:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:12,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-19T21:40:12,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052472627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/7fdb857cdf50465fbc1449d656f4e58c is 50, key is test_row_0/A:col10/1732052412623/Put/seqid=0 2024-11-19T21:40:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052472628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052472630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741906_1082 (size=12301) 2024-11-19T21:40:12,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/7fdb857cdf50465fbc1449d656f4e58c 2024-11-19T21:40:12,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/b61bdcef19f94f86a299380e74646b40 is 50, key is test_row_0/B:col10/1732052412623/Put/seqid=0 2024-11-19T21:40:12,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741907_1083 (size=12301) 2024-11-19T21:40:12,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052472739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052472739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052472739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052472948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052472949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052472949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:13,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/b61bdcef19f94f86a299380e74646b40 2024-11-19T21:40:13,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/7c04e98122694b71afee8573671df0d0 is 50, key is test_row_0/C:col10/1732052412623/Put/seqid=0 2024-11-19T21:40:13,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741908_1084 (size=12301) 2024-11-19T21:40:13,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052473258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:13,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052473259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:13,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052473259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:13,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/7c04e98122694b71afee8573671df0d0 2024-11-19T21:40:13,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/7fdb857cdf50465fbc1449d656f4e58c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7fdb857cdf50465fbc1449d656f4e58c 2024-11-19T21:40:13,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7fdb857cdf50465fbc1449d656f4e58c, entries=150, sequenceid=360, filesize=12.0 K 2024-11-19T21:40:13,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/b61bdcef19f94f86a299380e74646b40 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b61bdcef19f94f86a299380e74646b40 2024-11-19T21:40:13,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b61bdcef19f94f86a299380e74646b40, entries=150, sequenceid=360, filesize=12.0 K 2024-11-19T21:40:13,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/7c04e98122694b71afee8573671df0d0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/7c04e98122694b71afee8573671df0d0 2024-11-19T21:40:13,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/7c04e98122694b71afee8573671df0d0, entries=150, sequenceid=360, filesize=12.0 K 2024-11-19T21:40:13,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 7f292f8e50ece577fd7cb587fbc66fc6 in 928ms, sequenceid=360, compaction requested=true 2024-11-19T21:40:13,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:13,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:13,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:13,551 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:13,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:13,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:13,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:13,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T21:40:13,552 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:13,554 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:13,554 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:13,554 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:13,554 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d0e59a4d5f8042fe99223aba6d3d2d70, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/1587a5b29ece4d08af46cb80cab99f39, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8e9478d7dc6f49deb5aa6de9cf80c60e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7fdb857cdf50465fbc1449d656f4e58c] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=48.7 K 2024-11-19T21:40:13,555 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:13,555 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:13,555 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:13,556 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4995767f532c4c47b302a7a3d225760a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2067da6f6fd846a39f31f33b4953c070, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3a5f27e7c09c4bd3bf8f605bb69f0237, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b61bdcef19f94f86a299380e74646b40] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=48.7 K 2024-11-19T21:40:13,556 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e59a4d5f8042fe99223aba6d3d2d70, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732052408850 2024-11-19T21:40:13,557 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 4995767f532c4c47b302a7a3d225760a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732052408850 2024-11-19T21:40:13,557 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1587a5b29ece4d08af46cb80cab99f39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732052409583 2024-11-19T21:40:13,557 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2067da6f6fd846a39f31f33b4953c070, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732052409583 2024-11-19T21:40:13,558 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a5f27e7c09c4bd3bf8f605bb69f0237, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732052410234 2024-11-19T21:40:13,558 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9478d7dc6f49deb5aa6de9cf80c60e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732052410234 2024-11-19T21:40:13,559 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b61bdcef19f94f86a299380e74646b40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732052411477 2024-11-19T21:40:13,559 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fdb857cdf50465fbc1449d656f4e58c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732052411477 2024-11-19T21:40:13,581 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:13,582 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/417878b30fda4173b997ed85e759743f is 50, key is test_row_0/A:col10/1732052412623/Put/seqid=0 2024-11-19T21:40:13,593 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:13,594 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/b390688f155d4ccda81b9e298a557b5e is 50, key is test_row_0/B:col10/1732052412623/Put/seqid=0 2024-11-19T21:40:13,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741909_1085 (size=13119) 2024-11-19T21:40:13,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741910_1086 (size=13119) 2024-11-19T21:40:13,628 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/417878b30fda4173b997ed85e759743f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/417878b30fda4173b997ed85e759743f 2024-11-19T21:40:13,647 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into 417878b30fda4173b997ed85e759743f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:13,648 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:13,648 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=12, startTime=1732052413551; duration=0sec 2024-11-19T21:40:13,648 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:13,649 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:13,649 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:13,656 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:13,656 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:13,657 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:13,657 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/50c8e9ab17a643e3addd573163928bda, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/da81f3bde3ae4c88b8646786236ae8e0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bad9a8e72c1a49c7a0425a6f5a3d04e3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/7c04e98122694b71afee8573671df0d0] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=48.7 K 2024-11-19T21:40:13,658 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c8e9ab17a643e3addd573163928bda, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732052408850 2024-11-19T21:40:13,658 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting da81f3bde3ae4c88b8646786236ae8e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732052409583 2024-11-19T21:40:13,659 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad9a8e72c1a49c7a0425a6f5a3d04e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732052410234 2024-11-19T21:40:13,663 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c04e98122694b71afee8573671df0d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732052411477 2024-11-19T21:40:13,701 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:13,702 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/2478a8c3827e487685c24ece3973657b is 50, key is test_row_0/C:col10/1732052412623/Put/seqid=0 2024-11-19T21:40:13,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741911_1087 (size=13119) 2024-11-19T21:40:13,778 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:13,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:40:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:13,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e75adc8b9bd0423ebe7aa6265efd5565 is 50, key is test_row_0/A:col10/1732052413777/Put/seqid=0 2024-11-19T21:40:13,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741912_1088 (size=12301) 2024-11-19T21:40:13,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e75adc8b9bd0423ebe7aa6265efd5565 2024-11-19T21:40:13,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/da094cec2b024764a825ae7bdc304b15 is 50, key is test_row_0/B:col10/1732052413777/Put/seqid=0 2024-11-19T21:40:13,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741913_1089 (size=12301) 2024-11-19T21:40:13,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/da094cec2b024764a825ae7bdc304b15 2024-11-19T21:40:13,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/bd33b0e40cd9430883e3d129b7f9df86 is 50, key is test_row_0/C:col10/1732052413777/Put/seqid=0 2024-11-19T21:40:13,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052473861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052473861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052473861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:13,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741914_1090 (size=12301) 2024-11-19T21:40:13,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/bd33b0e40cd9430883e3d129b7f9df86 2024-11-19T21:40:13,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e75adc8b9bd0423ebe7aa6265efd5565 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e75adc8b9bd0423ebe7aa6265efd5565 2024-11-19T21:40:13,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e75adc8b9bd0423ebe7aa6265efd5565, entries=150, sequenceid=372, filesize=12.0 K 2024-11-19T21:40:13,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/da094cec2b024764a825ae7bdc304b15 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/da094cec2b024764a825ae7bdc304b15 2024-11-19T21:40:13,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/da094cec2b024764a825ae7bdc304b15, entries=150, sequenceid=372, filesize=12.0 K 2024-11-19T21:40:13,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/bd33b0e40cd9430883e3d129b7f9df86 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bd33b0e40cd9430883e3d129b7f9df86 2024-11-19T21:40:13,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bd33b0e40cd9430883e3d129b7f9df86, entries=150, sequenceid=372, filesize=12.0 K 2024-11-19T21:40:13,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7f292f8e50ece577fd7cb587fbc66fc6 in 127ms, sequenceid=372, compaction requested=false 2024-11-19T21:40:13,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:13,975 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:13,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T21:40:13,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/6811cba114414ad6a9285a31827c63c7 is 50, key is test_row_0/A:col10/1732052413973/Put/seqid=0 2024-11-19T21:40:14,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052474005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052474006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052474006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741915_1091 (size=14741) 2024-11-19T21:40:14,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/6811cba114414ad6a9285a31827c63c7 2024-11-19T21:40:14,044 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/b390688f155d4ccda81b9e298a557b5e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b390688f155d4ccda81b9e298a557b5e 2024-11-19T21:40:14,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/519bd8e07fa749bb9d3c9c9fac4cf16d is 50, key is test_row_0/B:col10/1732052413973/Put/seqid=0 2024-11-19T21:40:14,057 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into b390688f155d4ccda81b9e298a557b5e(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:14,057 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:14,057 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=12, startTime=1732052413551; duration=0sec 2024-11-19T21:40:14,057 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:14,057 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:14,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741916_1092 (size=12301) 2024-11-19T21:40:14,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/519bd8e07fa749bb9d3c9c9fac4cf16d 2024-11-19T21:40:14,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/48fe066e5d0f4f688cb863d70125db9e is 50, key is test_row_0/C:col10/1732052413973/Put/seqid=0 2024-11-19T21:40:14,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741917_1093 (size=12301) 2024-11-19T21:40:14,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/48fe066e5d0f4f688cb863d70125db9e 2024-11-19T21:40:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T21:40:14,118 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:14,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T21:40:14,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052474120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,123 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052474120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052474120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T21:40:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,125 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:14,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/6811cba114414ad6a9285a31827c63c7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/6811cba114414ad6a9285a31827c63c7 2024-11-19T21:40:14,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/6811cba114414ad6a9285a31827c63c7, entries=200, sequenceid=398, filesize=14.4 K 2024-11-19T21:40:14,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/519bd8e07fa749bb9d3c9c9fac4cf16d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/519bd8e07fa749bb9d3c9c9fac4cf16d 2024-11-19T21:40:14,156 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/2478a8c3827e487685c24ece3973657b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2478a8c3827e487685c24ece3973657b 2024-11-19T21:40:14,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/519bd8e07fa749bb9d3c9c9fac4cf16d, entries=150, sequenceid=398, filesize=12.0 K 2024-11-19T21:40:14,167 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into 2478a8c3827e487685c24ece3973657b(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:14,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/48fe066e5d0f4f688cb863d70125db9e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/48fe066e5d0f4f688cb863d70125db9e 2024-11-19T21:40:14,167 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:14,167 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=12, startTime=1732052413551; duration=0sec 2024-11-19T21:40:14,167 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:14,167 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:14,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/48fe066e5d0f4f688cb863d70125db9e, entries=150, sequenceid=398, filesize=12.0 K 2024-11-19T21:40:14,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 7f292f8e50ece577fd7cb587fbc66fc6 in 206ms, sequenceid=398, compaction requested=true 2024-11-19T21:40:14,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:14,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:14,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:14,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:14,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T21:40:14,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:14,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T21:40:14,182 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:14,183 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:14,191 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:14,191 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:14,191 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:14,191 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:14,191 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:14,191 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:14,192 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2478a8c3827e487685c24ece3973657b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bd33b0e40cd9430883e3d129b7f9df86, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/48fe066e5d0f4f688cb863d70125db9e] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=36.8 K 2024-11-19T21:40:14,192 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/417878b30fda4173b997ed85e759743f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e75adc8b9bd0423ebe7aa6265efd5565, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/6811cba114414ad6a9285a31827c63c7] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=39.2 K 2024-11-19T21:40:14,192 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2478a8c3827e487685c24ece3973657b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732052411477 2024-11-19T21:40:14,193 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting bd33b0e40cd9430883e3d129b7f9df86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732052412626 2024-11-19T21:40:14,193 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 48fe066e5d0f4f688cb863d70125db9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732052413844 2024-11-19T21:40:14,195 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417878b30fda4173b997ed85e759743f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732052411477 2024-11-19T21:40:14,196 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting e75adc8b9bd0423ebe7aa6265efd5565, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732052412626 2024-11-19T21:40:14,197 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6811cba114414ad6a9285a31827c63c7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732052413844 2024-11-19T21:40:14,210 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:14,211 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/d138afaf3cee4ecaa646da83dbba5cdf is 50, key is test_row_0/C:col10/1732052413973/Put/seqid=0 2024-11-19T21:40:14,217 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:14,217 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/fe67a8da36864c999058ded94ad37488 is 50, key is test_row_0/A:col10/1732052413973/Put/seqid=0 2024-11-19T21:40:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T21:40:14,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741919_1095 (size=13221) 2024-11-19T21:40:14,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741918_1094 (size=13221) 2024-11-19T21:40:14,262 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/d138afaf3cee4ecaa646da83dbba5cdf as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/d138afaf3cee4ecaa646da83dbba5cdf 2024-11-19T21:40:14,270 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into d138afaf3cee4ecaa646da83dbba5cdf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:14,270 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:14,270 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=13, startTime=1732052414182; duration=0sec 2024-11-19T21:40:14,271 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:14,271 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:14,271 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:14,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T21:40:14,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:14,278 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:40:14,279 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:14,279 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:14,279 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:14,279 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b390688f155d4ccda81b9e298a557b5e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/da094cec2b024764a825ae7bdc304b15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/519bd8e07fa749bb9d3c9c9fac4cf16d] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=36.8 K 2024-11-19T21:40:14,280 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b390688f155d4ccda81b9e298a557b5e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732052411477 2024-11-19T21:40:14,281 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting da094cec2b024764a825ae7bdc304b15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732052412626 2024-11-19T21:40:14,282 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 519bd8e07fa749bb9d3c9c9fac4cf16d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732052413844 2024-11-19T21:40:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/3ce4fd2c67ab4a3495f3096976d6fbd8 is 50, key is test_row_0/A:col10/1732052414003/Put/seqid=0 2024-11-19T21:40:14,325 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:14,325 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/6c35624020234e44bfcb8f62ef43a56d is 50, key is test_row_0/B:col10/1732052413973/Put/seqid=0 2024-11-19T21:40:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:14,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:14,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741920_1096 (size=12301) 2024-11-19T21:40:14,345 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/3ce4fd2c67ab4a3495f3096976d6fbd8 2024-11-19T21:40:14,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/1641fc28e0aa45af8ec4c0e3f86f93f8 is 50, key is test_row_0/B:col10/1732052414003/Put/seqid=0 2024-11-19T21:40:14,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741921_1097 (size=13221) 2024-11-19T21:40:14,387 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/6c35624020234e44bfcb8f62ef43a56d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6c35624020234e44bfcb8f62ef43a56d 2024-11-19T21:40:14,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052474387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052474388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052474389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,411 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into 6c35624020234e44bfcb8f62ef43a56d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:14,411 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:14,411 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=13, startTime=1732052414182; duration=0sec 2024-11-19T21:40:14,411 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:14,411 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:14,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741922_1098 (size=12301) 2024-11-19T21:40:14,417 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/1641fc28e0aa45af8ec4c0e3f86f93f8 2024-11-19T21:40:14,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/147b923c04124897b09674b41438fe17 is 50, key is test_row_0/C:col10/1732052414003/Put/seqid=0 2024-11-19T21:40:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T21:40:14,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741923_1099 (size=12301) 2024-11-19T21:40:14,481 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/147b923c04124897b09674b41438fe17 2024-11-19T21:40:14,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/3ce4fd2c67ab4a3495f3096976d6fbd8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/3ce4fd2c67ab4a3495f3096976d6fbd8 2024-11-19T21:40:14,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052474499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052474510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052474510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,521 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/3ce4fd2c67ab4a3495f3096976d6fbd8, entries=150, sequenceid=412, filesize=12.0 K 2024-11-19T21:40:14,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/1641fc28e0aa45af8ec4c0e3f86f93f8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1641fc28e0aa45af8ec4c0e3f86f93f8 2024-11-19T21:40:14,529 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1641fc28e0aa45af8ec4c0e3f86f93f8, entries=150, sequenceid=412, filesize=12.0 K 2024-11-19T21:40:14,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/147b923c04124897b09674b41438fe17 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/147b923c04124897b09674b41438fe17 2024-11-19T21:40:14,539 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/147b923c04124897b09674b41438fe17, entries=150, sequenceid=412, filesize=12.0 K 2024-11-19T21:40:14,541 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7f292f8e50ece577fd7cb587fbc66fc6 in 263ms, sequenceid=412, compaction requested=false 2024-11-19T21:40:14,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:14,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:14,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T21:40:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T21:40:14,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T21:40:14,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 418 msec 2024-11-19T21:40:14,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 429 msec 2024-11-19T21:40:14,653 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/fe67a8da36864c999058ded94ad37488 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fe67a8da36864c999058ded94ad37488 2024-11-19T21:40:14,664 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into fe67a8da36864c999058ded94ad37488(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:14,664 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:14,664 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=13, startTime=1732052414182; duration=0sec 2024-11-19T21:40:14,664 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:14,665 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:14,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:14,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T21:40:14,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/f997e87c38fe426ead0019087a11928e is 50, key is test_row_0/A:col10/1732052414709/Put/seqid=0 2024-11-19T21:40:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052474727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052474727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052474731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741924_1100 (size=12301) 2024-11-19T21:40:14,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/f997e87c38fe426ead0019087a11928e 2024-11-19T21:40:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T21:40:14,748 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:14,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:14,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/a263cfce4ad04289bd10be7cba60286f is 50, key is test_row_0/B:col10/1732052414709/Put/seqid=0 2024-11-19T21:40:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T21:40:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T21:40:14,752 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:14,752 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:14,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:14,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741925_1101 (size=12301) 2024-11-19T21:40:14,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/a263cfce4ad04289bd10be7cba60286f 2024-11-19T21:40:14,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/99650f727b634e61a46ef9376834c36f is 50, key is test_row_0/C:col10/1732052414709/Put/seqid=0 2024-11-19T21:40:14,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741926_1102 (size=12301) 2024-11-19T21:40:14,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052474838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052474838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052474838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T21:40:14,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T21:40:14,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:14,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:14,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:14,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:14,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052475048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052475048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052475048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T21:40:15,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:15,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T21:40:15,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/99650f727b634e61a46ef9376834c36f 2024-11-19T21:40:15,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/f997e87c38fe426ead0019087a11928e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/f997e87c38fe426ead0019087a11928e 2024-11-19T21:40:15,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/f997e87c38fe426ead0019087a11928e, entries=150, sequenceid=440, filesize=12.0 K 2024-11-19T21:40:15,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/a263cfce4ad04289bd10be7cba60286f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a263cfce4ad04289bd10be7cba60286f 2024-11-19T21:40:15,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T21:40:15,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:15,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a263cfce4ad04289bd10be7cba60286f, entries=150, sequenceid=440, filesize=12.0 K 2024-11-19T21:40:15,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/99650f727b634e61a46ef9376834c36f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/99650f727b634e61a46ef9376834c36f 2024-11-19T21:40:15,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/99650f727b634e61a46ef9376834c36f, entries=150, sequenceid=440, filesize=12.0 K 2024-11-19T21:40:15,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7f292f8e50ece577fd7cb587fbc66fc6 in 528ms, sequenceid=440, compaction requested=true 2024-11-19T21:40:15,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:15,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:15,240 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:15,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:15,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:15,240 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:15,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:15,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:15,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:15,243 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:15,244 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:15,244 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,244 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6c35624020234e44bfcb8f62ef43a56d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1641fc28e0aa45af8ec4c0e3f86f93f8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a263cfce4ad04289bd10be7cba60286f] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=36.9 K 2024-11-19T21:40:15,244 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:15,244 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:15,244 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,244 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fe67a8da36864c999058ded94ad37488, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/3ce4fd2c67ab4a3495f3096976d6fbd8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/f997e87c38fe426ead0019087a11928e] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=36.9 K 2024-11-19T21:40:15,245 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c35624020234e44bfcb8f62ef43a56d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732052413844 2024-11-19T21:40:15,245 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe67a8da36864c999058ded94ad37488, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732052413844 2024-11-19T21:40:15,245 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 1641fc28e0aa45af8ec4c0e3f86f93f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732052413999 2024-11-19T21:40:15,245 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ce4fd2c67ab4a3495f3096976d6fbd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732052413999 2024-11-19T21:40:15,246 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting f997e87c38fe426ead0019087a11928e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732052414374 2024-11-19T21:40:15,246 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a263cfce4ad04289bd10be7cba60286f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732052414374 2024-11-19T21:40:15,257 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:15,258 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/caf9fbd57db8438090b51264c84e7c8b is 50, key is test_row_0/A:col10/1732052414709/Put/seqid=0 2024-11-19T21:40:15,276 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:15,277 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/54167cff7d174048b2393fc5147def64 is 50, key is test_row_0/B:col10/1732052414709/Put/seqid=0 2024-11-19T21:40:15,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741927_1103 (size=13323) 2024-11-19T21:40:15,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741928_1104 (size=13323) 2024-11-19T21:40:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:15,363 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:15,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:40:15,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T21:40:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:15,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:15,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/52ddfc5286824716989a27a13be6fe14 is 50, key is test_row_0/A:col10/1732052414725/Put/seqid=0 2024-11-19T21:40:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T21:40:15,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741929_1105 (size=14741) 2024-11-19T21:40:15,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/52ddfc5286824716989a27a13be6fe14 2024-11-19T21:40:15,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/c77c815d30274a7b9d325c68ed083bf8 is 50, key is test_row_0/B:col10/1732052414725/Put/seqid=0 2024-11-19T21:40:15,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052475420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741930_1106 (size=12301) 2024-11-19T21:40:15,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052475423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052475424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/c77c815d30274a7b9d325c68ed083bf8 2024-11-19T21:40:15,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/77f9405d4ab642caab9c7ea1dd792b73 is 50, key is test_row_0/C:col10/1732052414725/Put/seqid=0 2024-11-19T21:40:15,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741931_1107 (size=12301) 2024-11-19T21:40:15,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/77f9405d4ab642caab9c7ea1dd792b73 2024-11-19T21:40:15,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/52ddfc5286824716989a27a13be6fe14 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/52ddfc5286824716989a27a13be6fe14 2024-11-19T21:40:15,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/52ddfc5286824716989a27a13be6fe14, entries=200, sequenceid=453, filesize=14.4 K 2024-11-19T21:40:15,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/c77c815d30274a7b9d325c68ed083bf8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c77c815d30274a7b9d325c68ed083bf8 2024-11-19T21:40:15,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c77c815d30274a7b9d325c68ed083bf8, entries=150, sequenceid=453, filesize=12.0 K 2024-11-19T21:40:15,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/77f9405d4ab642caab9c7ea1dd792b73 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/77f9405d4ab642caab9c7ea1dd792b73 2024-11-19T21:40:15,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/77f9405d4ab642caab9c7ea1dd792b73, entries=150, sequenceid=453, filesize=12.0 K 2024-11-19T21:40:15,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 7f292f8e50ece577fd7cb587fbc66fc6 in 141ms, sequenceid=453, compaction requested=true 2024-11-19T21:40:15,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:15,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:15,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T21:40:15,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:15,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T21:40:15,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:15,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-19T21:40:15,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T21:40:15,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,522 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:40:15,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/7e35f01a0ba14ac089c18238eb2f6aab is 50, key is test_row_0/A:col10/1732052415421/Put/seqid=0 2024-11-19T21:40:15,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:15,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741932_1108 (size=12301) 2024-11-19T21:40:15,553 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/7e35f01a0ba14ac089c18238eb2f6aab 2024-11-19T21:40:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052475550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052475551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052475552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/03f51a4653104737a648742838b0b47e is 50, key is test_row_0/B:col10/1732052415421/Put/seqid=0 2024-11-19T21:40:15,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741933_1109 (size=12301) 2024-11-19T21:40:15,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052475658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052475668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052475669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,723 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/54167cff7d174048b2393fc5147def64 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/54167cff7d174048b2393fc5147def64 2024-11-19T21:40:15,723 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/caf9fbd57db8438090b51264c84e7c8b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/caf9fbd57db8438090b51264c84e7c8b 2024-11-19T21:40:15,729 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into 54167cff7d174048b2393fc5147def64(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:15,729 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:15,729 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=13, startTime=1732052415240; duration=0sec 2024-11-19T21:40:15,729 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into caf9fbd57db8438090b51264c84e7c8b(size=13.0 K), total size for store is 27.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:15,729 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:15,729 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-19T21:40:15,729 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=13, startTime=1732052415240; duration=0sec 2024-11-19T21:40:15,729 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:15,730 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-19T21:40:15,730 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:15,730 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:15,730 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:15,730 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:15,732 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:15,732 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:15,732 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-19T21:40:15,732 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:15,732 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T21:40:15,732 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T21:40:15,732 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. because compaction request was cancelled 2024-11-19T21:40:15,732 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/d138afaf3cee4ecaa646da83dbba5cdf, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/147b923c04124897b09674b41438fe17, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/99650f727b634e61a46ef9376834c36f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/77f9405d4ab642caab9c7ea1dd792b73] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=48.9 K 2024-11-19T21:40:15,732 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:15,732 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T21:40:15,733 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d138afaf3cee4ecaa646da83dbba5cdf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732052413844 2024-11-19T21:40:15,733 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 147b923c04124897b09674b41438fe17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732052413999 2024-11-19T21:40:15,734 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T21:40:15,734 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T21:40:15,734 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. because compaction request was cancelled 2024-11-19T21:40:15,734 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:15,734 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T21:40:15,734 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 99650f727b634e61a46ef9376834c36f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732052414374 2024-11-19T21:40:15,735 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f9405d4ab642caab9c7ea1dd792b73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732052414725 2024-11-19T21:40:15,735 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T21:40:15,735 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T21:40:15,735 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. because compaction request was cancelled 2024-11-19T21:40:15,736 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:15,745 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:15,746 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/b578f5b038ef458db7ca3610f8008888 is 50, key is test_row_0/C:col10/1732052414725/Put/seqid=0 2024-11-19T21:40:15,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741934_1110 (size=13357) 2024-11-19T21:40:15,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052475869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052475878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052475878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T21:40:15,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1732052475948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,954 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in 7f292f8e50ece577fd7cb587fbc66fc6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33376 deadline: 1732052475958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:15,960 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 7f292f8e50ece577fd7cb587fbc66fc6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:15,978 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/03f51a4653104737a648742838b0b47e 2024-11-19T21:40:15,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/1a9dd88135d34e649dd980f4a412d0de is 50, key is test_row_0/C:col10/1732052415421/Put/seqid=0 2024-11-19T21:40:16,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741935_1111 (size=12301) 2024-11-19T21:40:16,158 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/b578f5b038ef458db7ca3610f8008888 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b578f5b038ef458db7ca3610f8008888 2024-11-19T21:40:16,164 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into b578f5b038ef458db7ca3610f8008888(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:16,164 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:16,164 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=12, startTime=1732052415504; duration=0sec 2024-11-19T21:40:16,165 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:16,165 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:16,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052476178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:16,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052476198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:16,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052476198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:16,443 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/1a9dd88135d34e649dd980f4a412d0de 2024-11-19T21:40:16,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/7e35f01a0ba14ac089c18238eb2f6aab as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7e35f01a0ba14ac089c18238eb2f6aab 2024-11-19T21:40:16,456 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7e35f01a0ba14ac089c18238eb2f6aab, entries=150, sequenceid=477, filesize=12.0 K 2024-11-19T21:40:16,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/03f51a4653104737a648742838b0b47e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/03f51a4653104737a648742838b0b47e 2024-11-19T21:40:16,467 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/03f51a4653104737a648742838b0b47e, entries=150, sequenceid=477, filesize=12.0 K 2024-11-19T21:40:16,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/1a9dd88135d34e649dd980f4a412d0de as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/1a9dd88135d34e649dd980f4a412d0de 2024-11-19T21:40:16,477 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/1a9dd88135d34e649dd980f4a412d0de, entries=150, sequenceid=477, filesize=12.0 K 2024-11-19T21:40:16,479 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 7f292f8e50ece577fd7cb587fbc66fc6 in 957ms, sequenceid=477, compaction requested=true 2024-11-19T21:40:16,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:16,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:16,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T21:40:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T21:40:16,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T21:40:16,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7280 sec 2024-11-19T21:40:16,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7340 sec 2024-11-19T21:40:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:16,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:40:16,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/d5129e50f3794aacb82d7fd5d65643f9 is 50, key is test_row_0/A:col10/1732052415549/Put/seqid=0 2024-11-19T21:40:16,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741936_1112 (size=12297) 2024-11-19T21:40:16,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052476744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052476750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052476752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:16,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052476858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:16,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052476858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:16,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052476858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T21:40:16,898 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:16,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T21:40:16,902 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T21:40:16,903 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:16,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T21:40:17,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T21:40:17,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:17,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:17,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052477068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052477068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052477068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:17,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/d5129e50f3794aacb82d7fd5d65643f9 2024-11-19T21:40:17,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/c3e11d42a98d4c8d80f63af5eadc9dd7 is 50, key is test_row_0/B:col10/1732052415549/Put/seqid=0 2024-11-19T21:40:17,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741937_1113 (size=9857) 2024-11-19T21:40:17,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T21:40:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T21:40:17,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T21:40:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:17,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052477378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052477378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052477378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:17,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T21:40:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:17,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/c3e11d42a98d4c8d80f63af5eadc9dd7 2024-11-19T21:40:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T21:40:17,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/5771b706e59c4aa7ab9c7b681f8edf96 is 50, key is test_row_0/C:col10/1732052415549/Put/seqid=0 2024-11-19T21:40:17,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741938_1114 (size=9857) 2024-11-19T21:40:17,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/5771b706e59c4aa7ab9c7b681f8edf96 2024-11-19T21:40:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/d5129e50f3794aacb82d7fd5d65643f9 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d5129e50f3794aacb82d7fd5d65643f9 2024-11-19T21:40:17,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d5129e50f3794aacb82d7fd5d65643f9, entries=150, sequenceid=493, filesize=12.0 K 2024-11-19T21:40:17,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/c3e11d42a98d4c8d80f63af5eadc9dd7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c3e11d42a98d4c8d80f63af5eadc9dd7 2024-11-19T21:40:17,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c3e11d42a98d4c8d80f63af5eadc9dd7, entries=100, sequenceid=493, filesize=9.6 K 2024-11-19T21:40:17,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/5771b706e59c4aa7ab9c7b681f8edf96 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5771b706e59c4aa7ab9c7b681f8edf96 2024-11-19T21:40:17,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5771b706e59c4aa7ab9c7b681f8edf96, entries=100, sequenceid=493, filesize=9.6 K 2024-11-19T21:40:17,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 7f292f8e50ece577fd7cb587fbc66fc6 in 913ms, sequenceid=493, compaction requested=true 2024-11-19T21:40:17,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:17,605 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:17,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:17,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:17,606 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52662 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:17,606 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:17,606 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:17,606 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:17,607 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/caf9fbd57db8438090b51264c84e7c8b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/52ddfc5286824716989a27a13be6fe14, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7e35f01a0ba14ac089c18238eb2f6aab, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d5129e50f3794aacb82d7fd5d65643f9] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=51.4 K 2024-11-19T21:40:17,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:17,607 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting caf9fbd57db8438090b51264c84e7c8b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732052414374 2024-11-19T21:40:17,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:17,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:17,608 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ddfc5286824716989a27a13be6fe14, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732052414721 2024-11-19T21:40:17,609 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e35f01a0ba14ac089c18238eb2f6aab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732052415409 2024-11-19T21:40:17,609 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:17,609 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:17,609 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,609 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/54167cff7d174048b2393fc5147def64, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c77c815d30274a7b9d325c68ed083bf8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/03f51a4653104737a648742838b0b47e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c3e11d42a98d4c8d80f63af5eadc9dd7] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=46.7 K 2024-11-19T21:40:17,610 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5129e50f3794aacb82d7fd5d65643f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732052415549 2024-11-19T21:40:17,610 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 54167cff7d174048b2393fc5147def64, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732052414374 2024-11-19T21:40:17,611 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c77c815d30274a7b9d325c68ed083bf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732052414725 2024-11-19T21:40:17,611 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f51a4653104737a648742838b0b47e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732052415409 2024-11-19T21:40:17,612 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e11d42a98d4c8d80f63af5eadc9dd7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732052415549 2024-11-19T21:40:17,641 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:17,642 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/b8b72c4d7dbb4149925adea49718c0ac is 50, key is test_row_0/A:col10/1732052415549/Put/seqid=0 2024-11-19T21:40:17,658 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:17,658 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/fd25677e6a114d11aeacbe7d09de0943 is 50, key is test_row_0/B:col10/1732052415549/Put/seqid=0 2024-11-19T21:40:17,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T21:40:17,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:17,671 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:40:17,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741940_1116 (size=13459) 2024-11-19T21:40:17,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e3bba9f637b54024a9bf4b56e9bc85db is 50, key is test_row_0/A:col10/1732052416748/Put/seqid=0 2024-11-19T21:40:17,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741939_1115 (size=13459) 2024-11-19T21:40:17,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741941_1117 (size=12301) 2024-11-19T21:40:17,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:17,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052477898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:17,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052477902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052477904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:17,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052478008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:18,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052478009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052478009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T21:40:18,093 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/fd25677e6a114d11aeacbe7d09de0943 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fd25677e6a114d11aeacbe7d09de0943 2024-11-19T21:40:18,100 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e3bba9f637b54024a9bf4b56e9bc85db 2024-11-19T21:40:18,101 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into fd25677e6a114d11aeacbe7d09de0943(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:18,101 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:18,101 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=12, startTime=1732052417606; duration=0sec 2024-11-19T21:40:18,101 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:18,101 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:18,101 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:18,104 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:18,104 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:18,104 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:18,104 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b578f5b038ef458db7ca3610f8008888, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/1a9dd88135d34e649dd980f4a412d0de, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5771b706e59c4aa7ab9c7b681f8edf96] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=34.7 K 2024-11-19T21:40:18,105 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b578f5b038ef458db7ca3610f8008888, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732052414725 2024-11-19T21:40:18,106 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a9dd88135d34e649dd980f4a412d0de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732052415409 2024-11-19T21:40:18,106 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/b8b72c4d7dbb4149925adea49718c0ac as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b8b72c4d7dbb4149925adea49718c0ac 2024-11-19T21:40:18,107 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 5771b706e59c4aa7ab9c7b681f8edf96, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732052415549 2024-11-19T21:40:18,116 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into b8b72c4d7dbb4149925adea49718c0ac(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:18,116 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:18,116 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=12, startTime=1732052417605; duration=0sec 2024-11-19T21:40:18,116 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:18,116 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/acf4cf64280643b5ba30feff7f61e84b is 50, key is test_row_0/B:col10/1732052416748/Put/seqid=0 2024-11-19T21:40:18,121 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:18,122 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3d1a7a4270814b4c8d69cd81631048bd is 50, key is test_row_0/C:col10/1732052415549/Put/seqid=0 2024-11-19T21:40:18,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741942_1118 (size=12301) 2024-11-19T21:40:18,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741943_1119 (size=13459) 2024-11-19T21:40:18,143 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/3d1a7a4270814b4c8d69cd81631048bd as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d1a7a4270814b4c8d69cd81631048bd 2024-11-19T21:40:18,150 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into 3d1a7a4270814b4c8d69cd81631048bd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:18,151 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:18,151 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=13, startTime=1732052417607; duration=0sec 2024-11-19T21:40:18,151 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:18,151 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:18,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052478218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:18,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052478218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:18,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052478218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,530 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/acf4cf64280643b5ba30feff7f61e84b 2024-11-19T21:40:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052478528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:18,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052478528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:18,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052478528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/461345f909414fb6b1f2ff55d8b1f823 is 50, key is test_row_0/C:col10/1732052416748/Put/seqid=0 2024-11-19T21:40:18,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741944_1120 (size=12301) 2024-11-19T21:40:18,585 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/461345f909414fb6b1f2ff55d8b1f823 2024-11-19T21:40:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/e3bba9f637b54024a9bf4b56e9bc85db as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e3bba9f637b54024a9bf4b56e9bc85db 2024-11-19T21:40:18,608 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e3bba9f637b54024a9bf4b56e9bc85db, entries=150, sequenceid=517, filesize=12.0 K 2024-11-19T21:40:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/acf4cf64280643b5ba30feff7f61e84b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/acf4cf64280643b5ba30feff7f61e84b 2024-11-19T21:40:18,623 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/acf4cf64280643b5ba30feff7f61e84b, entries=150, sequenceid=517, filesize=12.0 K 2024-11-19T21:40:18,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/461345f909414fb6b1f2ff55d8b1f823 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/461345f909414fb6b1f2ff55d8b1f823 2024-11-19T21:40:18,636 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/461345f909414fb6b1f2ff55d8b1f823, entries=150, sequenceid=517, filesize=12.0 K 2024-11-19T21:40:18,637 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 7f292f8e50ece577fd7cb587fbc66fc6 in 966ms, sequenceid=517, compaction requested=false 2024-11-19T21:40:18,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:18,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:18,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T21:40:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T21:40:18,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T21:40:18,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7360 sec 2024-11-19T21:40:18,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.7440 sec 2024-11-19T21:40:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:19,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:19,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:40:19,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/02c0eb0ea8ac4faa98a08bb524f459cc is 50, key is test_row_0/A:col10/1732052417895/Put/seqid=0 2024-11-19T21:40:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T21:40:19,048 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:19,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-19T21:40:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T21:40:19,051 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:19,051 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:19,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:19,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741945_1121 (size=17181) 2024-11-19T21:40:19,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052479105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052479106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052479107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T21:40:19,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T21:40:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052479218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052479218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052479218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T21:40:19,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:19,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T21:40:19,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052479428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052479428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052479428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/02c0eb0ea8ac4faa98a08bb524f459cc 2024-11-19T21:40:19,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/efd31c3f16dc4392bf17cb81d020c2bf is 50, key is test_row_0/B:col10/1732052417895/Put/seqid=0 2024-11-19T21:40:19,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741946_1122 (size=12301) 2024-11-19T21:40:19,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T21:40:19,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T21:40:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:19,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T21:40:19,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052479738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052479748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052479748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:19,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:19,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T21:40:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:19,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/efd31c3f16dc4392bf17cb81d020c2bf 2024-11-19T21:40:19,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/187dc428577148a7b8803e0c2c70b853 is 50, key is test_row_0/C:col10/1732052417895/Put/seqid=0 2024-11-19T21:40:19,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741947_1123 (size=12301) 2024-11-19T21:40:19,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/187dc428577148a7b8803e0c2c70b853 2024-11-19T21:40:19,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/02c0eb0ea8ac4faa98a08bb524f459cc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/02c0eb0ea8ac4faa98a08bb524f459cc 2024-11-19T21:40:19,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/02c0eb0ea8ac4faa98a08bb524f459cc, entries=250, sequenceid=534, filesize=16.8 K 2024-11-19T21:40:19,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/efd31c3f16dc4392bf17cb81d020c2bf as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/efd31c3f16dc4392bf17cb81d020c2bf 2024-11-19T21:40:19,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/efd31c3f16dc4392bf17cb81d020c2bf, entries=150, sequenceid=534, filesize=12.0 K 2024-11-19T21:40:19,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/187dc428577148a7b8803e0c2c70b853 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/187dc428577148a7b8803e0c2c70b853 2024-11-19T21:40:19,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/187dc428577148a7b8803e0c2c70b853, entries=150, sequenceid=534, filesize=12.0 K 2024-11-19T21:40:19,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 7f292f8e50ece577fd7cb587fbc66fc6 in 894ms, sequenceid=534, compaction requested=true 2024-11-19T21:40:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:19,935 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:19,935 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:19,937 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:19,937 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:19,937 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:19,937 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:19,937 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,937 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,937 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fd25677e6a114d11aeacbe7d09de0943, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/acf4cf64280643b5ba30feff7f61e84b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/efd31c3f16dc4392bf17cb81d020c2bf] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=37.2 K 2024-11-19T21:40:19,937 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b8b72c4d7dbb4149925adea49718c0ac, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e3bba9f637b54024a9bf4b56e9bc85db, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/02c0eb0ea8ac4faa98a08bb524f459cc] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=41.9 K 2024-11-19T21:40:19,937 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting fd25677e6a114d11aeacbe7d09de0943, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732052415409 2024-11-19T21:40:19,938 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8b72c4d7dbb4149925adea49718c0ac, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732052415409 2024-11-19T21:40:19,938 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting acf4cf64280643b5ba30feff7f61e84b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1732052416738 2024-11-19T21:40:19,938 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3bba9f637b54024a9bf4b56e9bc85db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1732052416738 2024-11-19T21:40:19,938 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting efd31c3f16dc4392bf17cb81d020c2bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1732052417895 2024-11-19T21:40:19,938 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02c0eb0ea8ac4faa98a08bb524f459cc, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1732052417895 2024-11-19T21:40:19,953 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:19,954 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/644e5830ebf0427fbf81d7a3e6462c81 is 50, key is test_row_0/A:col10/1732052417895/Put/seqid=0 2024-11-19T21:40:19,954 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:19,955 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/5b188e732b52496380a902766fc4d4b2 is 50, key is test_row_0/B:col10/1732052417895/Put/seqid=0 2024-11-19T21:40:19,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T21:40:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:19,975 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T21:40:19,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741948_1124 (size=13561) 2024-11-19T21:40:19,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741949_1125 (size=13561) 2024-11-19T21:40:19,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/552d3adb70894c24987b695f3879e660 is 50, key is test_row_0/A:col10/1732052419104/Put/seqid=0 2024-11-19T21:40:20,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741950_1126 (size=12301) 2024-11-19T21:40:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T21:40:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:20,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. as already flushing 2024-11-19T21:40:20,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052480271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052480281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052480281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,389 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/644e5830ebf0427fbf81d7a3e6462c81 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/644e5830ebf0427fbf81d7a3e6462c81 2024-11-19T21:40:20,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052480388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052480388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,402 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/552d3adb70894c24987b695f3879e660 2024-11-19T21:40:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052480398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,407 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into 644e5830ebf0427fbf81d7a3e6462c81(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:20,407 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:20,407 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=13, startTime=1732052419935; duration=0sec 2024-11-19T21:40:20,407 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:20,407 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:20,407 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:20,411 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:20,411 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:20,411 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:20,411 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d1a7a4270814b4c8d69cd81631048bd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/461345f909414fb6b1f2ff55d8b1f823, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/187dc428577148a7b8803e0c2c70b853] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=37.2 K 2024-11-19T21:40:20,412 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/5b188e732b52496380a902766fc4d4b2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/5b188e732b52496380a902766fc4d4b2 2024-11-19T21:40:20,412 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d1a7a4270814b4c8d69cd81631048bd, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732052415409 2024-11-19T21:40:20,414 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 461345f909414fb6b1f2ff55d8b1f823, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1732052416738 2024-11-19T21:40:20,416 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 187dc428577148a7b8803e0c2c70b853, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1732052417895 2024-11-19T21:40:20,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/2a72b83622ae4b29a6cfc31aa409957c is 50, key is test_row_0/B:col10/1732052419104/Put/seqid=0 2024-11-19T21:40:20,426 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into 5b188e732b52496380a902766fc4d4b2(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:20,426 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:20,426 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=13, startTime=1732052419935; duration=0sec 2024-11-19T21:40:20,426 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:20,426 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:20,438 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:20,439 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/b9bddd56d5d145e2a26054c613427b10 is 50, key is test_row_0/C:col10/1732052417895/Put/seqid=0 2024-11-19T21:40:20,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741951_1127 (size=12301) 2024-11-19T21:40:20,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741952_1128 (size=13561) 2024-11-19T21:40:20,454 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/b9bddd56d5d145e2a26054c613427b10 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b9bddd56d5d145e2a26054c613427b10 2024-11-19T21:40:20,460 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into b9bddd56d5d145e2a26054c613427b10(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:20,460 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:20,460 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=13, startTime=1732052419935; duration=0sec 2024-11-19T21:40:20,460 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:20,460 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:20,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052480598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052480598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052480608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,843 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/2a72b83622ae4b29a6cfc31aa409957c 2024-11-19T21:40:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/0dd9c87520f348be93329405a26fc965 is 50, key is test_row_0/C:col10/1732052419104/Put/seqid=0 2024-11-19T21:40:20,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741953_1129 (size=12301) 2024-11-19T21:40:20,855 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/0dd9c87520f348be93329405a26fc965 2024-11-19T21:40:20,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/552d3adb70894c24987b695f3879e660 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/552d3adb70894c24987b695f3879e660 2024-11-19T21:40:20,865 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/552d3adb70894c24987b695f3879e660, entries=150, sequenceid=556, filesize=12.0 K 2024-11-19T21:40:20,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/2a72b83622ae4b29a6cfc31aa409957c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2a72b83622ae4b29a6cfc31aa409957c 2024-11-19T21:40:20,872 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2a72b83622ae4b29a6cfc31aa409957c, entries=150, sequenceid=556, filesize=12.0 K 2024-11-19T21:40:20,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/0dd9c87520f348be93329405a26fc965 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0dd9c87520f348be93329405a26fc965 2024-11-19T21:40:20,877 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0dd9c87520f348be93329405a26fc965, entries=150, sequenceid=556, filesize=12.0 K 2024-11-19T21:40:20,878 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 7f292f8e50ece577fd7cb587fbc66fc6 in 903ms, sequenceid=556, compaction requested=false 2024-11-19T21:40:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:20,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-19T21:40:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-19T21:40:20,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T21:40:20,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-11-19T21:40:20,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.8360 sec 2024-11-19T21:40:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:20,910 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:20,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T21:40:20,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/7aba48119f884b1d9f0a1b6bddec12ce is 50, key is test_row_0/A:col10/1732052420267/Put/seqid=0 2024-11-19T21:40:20,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741954_1130 (size=17181) 2024-11-19T21:40:20,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052480932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052480933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:20,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052480936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052481038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052481048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052481049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,162 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T21:40:21,163 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:21,164 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T21:40:21,165 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:21,166 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T21:40:21,166 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:21,167 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:21,167 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:21,167 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:21,168 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:21,168 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:21,168 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:21,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:21,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:21,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:21,170 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T21:40:21,170 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:21,170 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:21,170 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:21,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T21:40:21,199 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:21,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052481248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:21,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052481258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052481258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/7aba48119f884b1d9f0a1b6bddec12ce 2024-11-19T21:40:21,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/10a1e843aaed4ab7a4943841ca79a51c is 50, key is test_row_0/B:col10/1732052420267/Put/seqid=0 2024-11-19T21:40:21,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741955_1131 (size=12301) 2024-11-19T21:40:21,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052481558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:21,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052481568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052481568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:21,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/10a1e843aaed4ab7a4943841ca79a51c 2024-11-19T21:40:21,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/deaf790edc034db2b750ac1505f9cb78 is 50, key is test_row_0/C:col10/1732052420267/Put/seqid=0 2024-11-19T21:40:21,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741956_1132 (size=12301) 2024-11-19T21:40:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33374 deadline: 1732052482069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:22,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33394 deadline: 1732052482079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33400 deadline: 1732052482079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:40:22,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/deaf790edc034db2b750ac1505f9cb78 2024-11-19T21:40:22,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/7aba48119f884b1d9f0a1b6bddec12ce as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7aba48119f884b1d9f0a1b6bddec12ce 2024-11-19T21:40:22,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7aba48119f884b1d9f0a1b6bddec12ce, entries=250, sequenceid=576, filesize=16.8 K 2024-11-19T21:40:22,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/10a1e843aaed4ab7a4943841ca79a51c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/10a1e843aaed4ab7a4943841ca79a51c 2024-11-19T21:40:22,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/10a1e843aaed4ab7a4943841ca79a51c, entries=150, sequenceid=576, filesize=12.0 K 2024-11-19T21:40:22,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/deaf790edc034db2b750ac1505f9cb78 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/deaf790edc034db2b750ac1505f9cb78 2024-11-19T21:40:22,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/deaf790edc034db2b750ac1505f9cb78, entries=150, sequenceid=576, filesize=12.0 K 2024-11-19T21:40:22,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 7f292f8e50ece577fd7cb587fbc66fc6 in 1285ms, sequenceid=576, compaction requested=true 2024-11-19T21:40:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:22,196 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f292f8e50ece577fd7cb587fbc66fc6:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:22,196 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:22,197 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:22,197 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:22,197 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/B is initiating minor compaction (all files) 2024-11-19T21:40:22,197 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/A is initiating minor compaction (all files) 2024-11-19T21:40:22,197 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/B in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:22,197 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/A in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:22,197 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/644e5830ebf0427fbf81d7a3e6462c81, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/552d3adb70894c24987b695f3879e660, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7aba48119f884b1d9f0a1b6bddec12ce] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=42.0 K 2024-11-19T21:40:22,197 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/5b188e732b52496380a902766fc4d4b2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2a72b83622ae4b29a6cfc31aa409957c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/10a1e843aaed4ab7a4943841ca79a51c] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=37.3 K 2024-11-19T21:40:22,198 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 644e5830ebf0427fbf81d7a3e6462c81, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1732052417895 2024-11-19T21:40:22,198 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b188e732b52496380a902766fc4d4b2, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1732052417895 2024-11-19T21:40:22,198 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a72b83622ae4b29a6cfc31aa409957c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732052419096 2024-11-19T21:40:22,198 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 552d3adb70894c24987b695f3879e660, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732052419096 2024-11-19T21:40:22,198 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 10a1e843aaed4ab7a4943841ca79a51c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1732052420267 2024-11-19T21:40:22,198 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aba48119f884b1d9f0a1b6bddec12ce, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1732052420267 2024-11-19T21:40:22,207 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#B#compaction#120 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:22,208 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/35a58d0d75f34f27904ffbfa2d53166d is 50, key is test_row_0/B:col10/1732052420267/Put/seqid=0 2024-11-19T21:40:22,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741957_1133 (size=13663) 2024-11-19T21:40:22,237 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#A#compaction#121 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:22,237 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/ad8ec27213114d21994ab12cc4fa80d2 is 50, key is test_row_0/A:col10/1732052420267/Put/seqid=0 2024-11-19T21:40:22,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741958_1134 (size=13663) 2024-11-19T21:40:22,276 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T21:40:22,619 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/35a58d0d75f34f27904ffbfa2d53166d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/35a58d0d75f34f27904ffbfa2d53166d 2024-11-19T21:40:22,625 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/B of 7f292f8e50ece577fd7cb587fbc66fc6 into 35a58d0d75f34f27904ffbfa2d53166d(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:22,625 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:22,625 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/B, priority=13, startTime=1732052422196; duration=0sec 2024-11-19T21:40:22,626 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:22,626 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:B 2024-11-19T21:40:22,626 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:22,627 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:22,627 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 7f292f8e50ece577fd7cb587fbc66fc6/C is initiating minor compaction (all files) 2024-11-19T21:40:22,627 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f292f8e50ece577fd7cb587fbc66fc6/C in TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:22,627 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b9bddd56d5d145e2a26054c613427b10, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0dd9c87520f348be93329405a26fc965, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/deaf790edc034db2b750ac1505f9cb78] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp, totalSize=37.3 K 2024-11-19T21:40:22,627 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b9bddd56d5d145e2a26054c613427b10, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1732052417895 2024-11-19T21:40:22,628 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd9c87520f348be93329405a26fc965, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1732052419096 2024-11-19T21:40:22,628 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting deaf790edc034db2b750ac1505f9cb78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1732052420267 2024-11-19T21:40:22,638 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f292f8e50ece577fd7cb587fbc66fc6#C#compaction#122 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:22,638 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/f2909afe3a8e40b28a5217f75a4d74ec is 50, key is test_row_0/C:col10/1732052420267/Put/seqid=0 2024-11-19T21:40:22,650 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/ad8ec27213114d21994ab12cc4fa80d2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/ad8ec27213114d21994ab12cc4fa80d2 2024-11-19T21:40:22,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741959_1135 (size=13663) 2024-11-19T21:40:22,657 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/A of 7f292f8e50ece577fd7cb587fbc66fc6 into ad8ec27213114d21994ab12cc4fa80d2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:22,658 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:22,658 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/A, priority=13, startTime=1732052422196; duration=0sec 2024-11-19T21:40:22,658 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:22,658 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:A 2024-11-19T21:40:22,659 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/f2909afe3a8e40b28a5217f75a4d74ec as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/f2909afe3a8e40b28a5217f75a4d74ec 2024-11-19T21:40:22,665 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f292f8e50ece577fd7cb587fbc66fc6/C of 7f292f8e50ece577fd7cb587fbc66fc6 into f2909afe3a8e40b28a5217f75a4d74ec(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:22,665 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:22,665 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., storeName=7f292f8e50ece577fd7cb587fbc66fc6/C, priority=13, startTime=1732052422196; duration=0sec 2024-11-19T21:40:22,665 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:22,666 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f292f8e50ece577fd7cb587fbc66fc6:C 2024-11-19T21:40:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:23,090 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:23,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T21:40:23,090 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T21:40:23,090 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T21:40:23,090 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:23,090 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:23,090 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:23,090 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:23,090 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:23,090 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:23,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:23,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:23,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/477b2fc2a6dc4c419ab53793b1cbc20a is 50, key is test_row_0/A:col10/1732052420935/Put/seqid=0 2024-11-19T21:40:23,100 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T21:40:23,100 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:23,100 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:23,100 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:23,100 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:23,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741960_1136 (size=12301) 2024-11-19T21:40:23,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/477b2fc2a6dc4c419ab53793b1cbc20a 2024-11-19T21:40:23,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/3e5c75e5f12e475ebeffbcc2f09cd108 is 50, key is test_row_0/B:col10/1732052420935/Put/seqid=0 2024-11-19T21:40:23,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741961_1137 (size=12301) 2024-11-19T21:40:23,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/3e5c75e5f12e475ebeffbcc2f09cd108 2024-11-19T21:40:23,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/ca13f699fa384c5a831579b087d072ac is 50, key is test_row_0/C:col10/1732052420935/Put/seqid=0 2024-11-19T21:40:23,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741962_1138 (size=12301) 2024-11-19T21:40:24,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/ca13f699fa384c5a831579b087d072ac 2024-11-19T21:40:24,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/477b2fc2a6dc4c419ab53793b1cbc20a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/477b2fc2a6dc4c419ab53793b1cbc20a 2024-11-19T21:40:24,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/477b2fc2a6dc4c419ab53793b1cbc20a, entries=150, sequenceid=600, filesize=12.0 K 2024-11-19T21:40:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/3e5c75e5f12e475ebeffbcc2f09cd108 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3e5c75e5f12e475ebeffbcc2f09cd108 2024-11-19T21:40:24,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3e5c75e5f12e475ebeffbcc2f09cd108, entries=150, sequenceid=600, filesize=12.0 K 2024-11-19T21:40:24,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/ca13f699fa384c5a831579b087d072ac as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/ca13f699fa384c5a831579b087d072ac 2024-11-19T21:40:24,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/ca13f699fa384c5a831579b087d072ac, entries=150, sequenceid=600, filesize=12.0 K 2024-11-19T21:40:24,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=6.71 KB/6870 for 7f292f8e50ece577fd7cb587fbc66fc6 in 1294ms, sequenceid=600, compaction requested=false 2024-11-19T21:40:24,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f292f8e50ece577fd7cb587fbc66fc6: 2024-11-19T21:40:26,010 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T21:40:26,010 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:26,010 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:26,010 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:26,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:26,030 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T21:40:26,030 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:26,030 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:26,030 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:26,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:26,030 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 133 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3202 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3154 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1534 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4595 rows 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1496 2024-11-19T21:40:26,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4482 rows 2024-11-19T21:40:26,031 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:40:26,031 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:40:26,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:26,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:26,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:26,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T21:40:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T21:40:26,044 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052426043"}]},"ts":"1732052426043"} 2024-11-19T21:40:26,046 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T21:40:26,046 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T21:40:26,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T21:40:26,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f292f8e50ece577fd7cb587fbc66fc6, UNASSIGN}] 2024-11-19T21:40:26,053 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f292f8e50ece577fd7cb587fbc66fc6, UNASSIGN 2024-11-19T21:40:26,055 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7f292f8e50ece577fd7cb587fbc66fc6, regionState=CLOSING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:26,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f292f8e50ece577fd7cb587fbc66fc6, UNASSIGN because future has completed 2024-11-19T21:40:26,057 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T21:40:26,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:40:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T21:40:26,214 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:26,214 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T21:40:26,215 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 7f292f8e50ece577fd7cb587fbc66fc6, disabling compactions & flushes 2024-11-19T21:40:26,215 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:26,215 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:26,215 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. after waiting 0 ms 2024-11-19T21:40:26,215 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:26,215 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 7f292f8e50ece577fd7cb587fbc66fc6 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-19T21:40:26,219 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/98669b7b5e824bfe82b213fc1c4f54c9 is 50, key is test_row_0/A:col10/1732052426029/Put/seqid=0 2024-11-19T21:40:26,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741963_1139 (size=12301) 2024-11-19T21:40:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T21:40:26,626 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/98669b7b5e824bfe82b213fc1c4f54c9 2024-11-19T21:40:26,634 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/0825f876f4624d82b102dc093d1dfb3c is 50, key is test_row_0/B:col10/1732052426029/Put/seqid=0 2024-11-19T21:40:26,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741964_1140 (size=12301) 2024-11-19T21:40:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T21:40:27,039 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/0825f876f4624d82b102dc093d1dfb3c 2024-11-19T21:40:27,050 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/b23a8bdd51be4b2ea4b4dde84f923aeb is 50, key is test_row_0/C:col10/1732052426029/Put/seqid=0 2024-11-19T21:40:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741965_1141 (size=12301) 2024-11-19T21:40:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T21:40:27,458 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/b23a8bdd51be4b2ea4b4dde84f923aeb 2024-11-19T21:40:27,464 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/A/98669b7b5e824bfe82b213fc1c4f54c9 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/98669b7b5e824bfe82b213fc1c4f54c9 2024-11-19T21:40:27,469 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/98669b7b5e824bfe82b213fc1c4f54c9, entries=150, sequenceid=606, filesize=12.0 K 2024-11-19T21:40:27,470 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/B/0825f876f4624d82b102dc093d1dfb3c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0825f876f4624d82b102dc093d1dfb3c 2024-11-19T21:40:27,475 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0825f876f4624d82b102dc093d1dfb3c, entries=150, sequenceid=606, filesize=12.0 K 2024-11-19T21:40:27,476 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/.tmp/C/b23a8bdd51be4b2ea4b4dde84f923aeb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b23a8bdd51be4b2ea4b4dde84f923aeb 2024-11-19T21:40:27,481 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b23a8bdd51be4b2ea4b4dde84f923aeb, entries=150, sequenceid=606, filesize=12.0 K 2024-11-19T21:40:27,482 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 7f292f8e50ece577fd7cb587fbc66fc6 in 1267ms, sequenceid=606, compaction requested=true 2024-11-19T21:40:27,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/c537bd317d784ed69e961bcb357b8246, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e7831ae6c590489e8425ca9944c733b9, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8f420f4e75384407a60d727f7ed2194a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/cce459d62c9e4134929944717f578a68, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e1e426681e9946d8b07a15deb7fbe30d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8035087a25434fccac7bd89f2179d053, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e94b4ee5629c4f4193465904ec35c515, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9571f98a5fc2468584d1bb2e03acbb1a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9726383dc7a7418f95bd1d540b11b1bd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b5e99d9ceca24d7382af2d04d8780452, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/67fc3bf860b348c5a899c6e09910711b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fb2f872de0904635882fe94c07ec8294, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/ef6925d2a67c4fc58a1c345e634684bc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/a1394b1783b64ff6a5773502fb2b1cef, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/08eb804efc224da0ab754e789bd0c4e4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/91178f42241e448b856c0f2e5b5644f9, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/58bf11fb1af945f782b55e13c18647c2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/0988c363d28d4f3aa3f477502b2b7053, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/123e319ccc2843c1bdf2a814aa8f1907, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/56ccf5abcfba461da0eb83bf29fc4dac, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d0e59a4d5f8042fe99223aba6d3d2d70, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/1587a5b29ece4d08af46cb80cab99f39, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8e9478d7dc6f49deb5aa6de9cf80c60e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/417878b30fda4173b997ed85e759743f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7fdb857cdf50465fbc1449d656f4e58c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e75adc8b9bd0423ebe7aa6265efd5565, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/6811cba114414ad6a9285a31827c63c7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fe67a8da36864c999058ded94ad37488, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/3ce4fd2c67ab4a3495f3096976d6fbd8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/caf9fbd57db8438090b51264c84e7c8b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/f997e87c38fe426ead0019087a11928e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/52ddfc5286824716989a27a13be6fe14, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7e35f01a0ba14ac089c18238eb2f6aab, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b8b72c4d7dbb4149925adea49718c0ac, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d5129e50f3794aacb82d7fd5d65643f9, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e3bba9f637b54024a9bf4b56e9bc85db, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/02c0eb0ea8ac4faa98a08bb524f459cc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/644e5830ebf0427fbf81d7a3e6462c81, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/552d3adb70894c24987b695f3879e660, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7aba48119f884b1d9f0a1b6bddec12ce] to archive 2024-11-19T21:40:27,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:40:27,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/c537bd317d784ed69e961bcb357b8246 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/c537bd317d784ed69e961bcb357b8246 2024-11-19T21:40:27,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e7831ae6c590489e8425ca9944c733b9 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e7831ae6c590489e8425ca9944c733b9 2024-11-19T21:40:27,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8f420f4e75384407a60d727f7ed2194a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8f420f4e75384407a60d727f7ed2194a 2024-11-19T21:40:27,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/cce459d62c9e4134929944717f578a68 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/cce459d62c9e4134929944717f578a68 2024-11-19T21:40:27,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e1e426681e9946d8b07a15deb7fbe30d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e1e426681e9946d8b07a15deb7fbe30d 2024-11-19T21:40:27,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8035087a25434fccac7bd89f2179d053 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8035087a25434fccac7bd89f2179d053 2024-11-19T21:40:27,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e94b4ee5629c4f4193465904ec35c515 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e94b4ee5629c4f4193465904ec35c515 2024-11-19T21:40:27,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9571f98a5fc2468584d1bb2e03acbb1a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9571f98a5fc2468584d1bb2e03acbb1a 2024-11-19T21:40:27,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9726383dc7a7418f95bd1d540b11b1bd to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/9726383dc7a7418f95bd1d540b11b1bd 2024-11-19T21:40:27,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b5e99d9ceca24d7382af2d04d8780452 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b5e99d9ceca24d7382af2d04d8780452 2024-11-19T21:40:27,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/67fc3bf860b348c5a899c6e09910711b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/67fc3bf860b348c5a899c6e09910711b 2024-11-19T21:40:27,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fb2f872de0904635882fe94c07ec8294 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fb2f872de0904635882fe94c07ec8294 2024-11-19T21:40:27,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/ef6925d2a67c4fc58a1c345e634684bc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/ef6925d2a67c4fc58a1c345e634684bc 2024-11-19T21:40:27,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/a1394b1783b64ff6a5773502fb2b1cef to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/a1394b1783b64ff6a5773502fb2b1cef 2024-11-19T21:40:27,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/08eb804efc224da0ab754e789bd0c4e4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/08eb804efc224da0ab754e789bd0c4e4 2024-11-19T21:40:27,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/91178f42241e448b856c0f2e5b5644f9 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/91178f42241e448b856c0f2e5b5644f9 2024-11-19T21:40:27,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/58bf11fb1af945f782b55e13c18647c2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/58bf11fb1af945f782b55e13c18647c2 2024-11-19T21:40:27,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/0988c363d28d4f3aa3f477502b2b7053 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/0988c363d28d4f3aa3f477502b2b7053 2024-11-19T21:40:27,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/123e319ccc2843c1bdf2a814aa8f1907 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/123e319ccc2843c1bdf2a814aa8f1907 2024-11-19T21:40:27,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/56ccf5abcfba461da0eb83bf29fc4dac to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/56ccf5abcfba461da0eb83bf29fc4dac 2024-11-19T21:40:27,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d0e59a4d5f8042fe99223aba6d3d2d70 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d0e59a4d5f8042fe99223aba6d3d2d70 2024-11-19T21:40:27,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/1587a5b29ece4d08af46cb80cab99f39 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/1587a5b29ece4d08af46cb80cab99f39 2024-11-19T21:40:27,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8e9478d7dc6f49deb5aa6de9cf80c60e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/8e9478d7dc6f49deb5aa6de9cf80c60e 2024-11-19T21:40:27,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/417878b30fda4173b997ed85e759743f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/417878b30fda4173b997ed85e759743f 2024-11-19T21:40:27,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7fdb857cdf50465fbc1449d656f4e58c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7fdb857cdf50465fbc1449d656f4e58c 2024-11-19T21:40:27,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e75adc8b9bd0423ebe7aa6265efd5565 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e75adc8b9bd0423ebe7aa6265efd5565 2024-11-19T21:40:27,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/6811cba114414ad6a9285a31827c63c7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/6811cba114414ad6a9285a31827c63c7 2024-11-19T21:40:27,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fe67a8da36864c999058ded94ad37488 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/fe67a8da36864c999058ded94ad37488 2024-11-19T21:40:27,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/3ce4fd2c67ab4a3495f3096976d6fbd8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/3ce4fd2c67ab4a3495f3096976d6fbd8 2024-11-19T21:40:27,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/caf9fbd57db8438090b51264c84e7c8b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/caf9fbd57db8438090b51264c84e7c8b 2024-11-19T21:40:27,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/f997e87c38fe426ead0019087a11928e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/f997e87c38fe426ead0019087a11928e 2024-11-19T21:40:27,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/52ddfc5286824716989a27a13be6fe14 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/52ddfc5286824716989a27a13be6fe14 2024-11-19T21:40:27,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7e35f01a0ba14ac089c18238eb2f6aab to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7e35f01a0ba14ac089c18238eb2f6aab 2024-11-19T21:40:27,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b8b72c4d7dbb4149925adea49718c0ac to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/b8b72c4d7dbb4149925adea49718c0ac 2024-11-19T21:40:27,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d5129e50f3794aacb82d7fd5d65643f9 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/d5129e50f3794aacb82d7fd5d65643f9 2024-11-19T21:40:27,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e3bba9f637b54024a9bf4b56e9bc85db to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/e3bba9f637b54024a9bf4b56e9bc85db 2024-11-19T21:40:27,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/02c0eb0ea8ac4faa98a08bb524f459cc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/02c0eb0ea8ac4faa98a08bb524f459cc 2024-11-19T21:40:27,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/644e5830ebf0427fbf81d7a3e6462c81 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/644e5830ebf0427fbf81d7a3e6462c81 2024-11-19T21:40:27,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/552d3adb70894c24987b695f3879e660 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/552d3adb70894c24987b695f3879e660 2024-11-19T21:40:27,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7aba48119f884b1d9f0a1b6bddec12ce to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/7aba48119f884b1d9f0a1b6bddec12ce 2024-11-19T21:40:27,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4489f949efd841edb251a3032aa04c49, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a3d196969853431b9eaaf7e86ebb1381, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dfe19f71bf8a47bdba3902cb211a6058, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3459970beb454c7fa24792d489633a5a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/bb38a0f46b2b46babb68fda31a84fe88, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dd6deb199c1a4ac49803fe4f2d4b7fb9, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dbe9a443a82a4117ac44900732ca6f5a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fe5210c3b16a49b0a09c4f070f4995a1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/9770fc8d584d49069cf20ba08cfe9f15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6afd95c8aba642e1bc3aba2aa9bc7531, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1a1bf19c84c644cf8c3900e34e9879a3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/85eca0a4f11b42db94556e2ee337c450, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a4407ac222ca44c78b52eb1e3e41d5c0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/8f6b74996cab4ab4bc9386f5936cc1e3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/d22994a628cc46df94eb05949c12fbb0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4353d20206d54fb8a736404967ce3a1a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/66adef52b81d4ce1ab6bf8c60953aafb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/21f44f787e0348b4bb8b4bba1f8541b0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/aea1d28835114915af2c0d176239ec51, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4995767f532c4c47b302a7a3d225760a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0453ab769c694ab397542fee30a8351a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2067da6f6fd846a39f31f33b4953c070, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3a5f27e7c09c4bd3bf8f605bb69f0237, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b390688f155d4ccda81b9e298a557b5e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b61bdcef19f94f86a299380e74646b40, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/da094cec2b024764a825ae7bdc304b15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6c35624020234e44bfcb8f62ef43a56d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/519bd8e07fa749bb9d3c9c9fac4cf16d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1641fc28e0aa45af8ec4c0e3f86f93f8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/54167cff7d174048b2393fc5147def64, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a263cfce4ad04289bd10be7cba60286f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c77c815d30274a7b9d325c68ed083bf8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/03f51a4653104737a648742838b0b47e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fd25677e6a114d11aeacbe7d09de0943, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c3e11d42a98d4c8d80f63af5eadc9dd7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/acf4cf64280643b5ba30feff7f61e84b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/5b188e732b52496380a902766fc4d4b2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/efd31c3f16dc4392bf17cb81d020c2bf, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2a72b83622ae4b29a6cfc31aa409957c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/10a1e843aaed4ab7a4943841ca79a51c] to archive 2024-11-19T21:40:27,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:40:27,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4489f949efd841edb251a3032aa04c49 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4489f949efd841edb251a3032aa04c49 2024-11-19T21:40:27,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a3d196969853431b9eaaf7e86ebb1381 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a3d196969853431b9eaaf7e86ebb1381 2024-11-19T21:40:27,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dfe19f71bf8a47bdba3902cb211a6058 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dfe19f71bf8a47bdba3902cb211a6058 2024-11-19T21:40:27,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3459970beb454c7fa24792d489633a5a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3459970beb454c7fa24792d489633a5a 2024-11-19T21:40:27,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/bb38a0f46b2b46babb68fda31a84fe88 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/bb38a0f46b2b46babb68fda31a84fe88 2024-11-19T21:40:27,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dd6deb199c1a4ac49803fe4f2d4b7fb9 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dd6deb199c1a4ac49803fe4f2d4b7fb9 2024-11-19T21:40:27,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dbe9a443a82a4117ac44900732ca6f5a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/dbe9a443a82a4117ac44900732ca6f5a 2024-11-19T21:40:27,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fe5210c3b16a49b0a09c4f070f4995a1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fe5210c3b16a49b0a09c4f070f4995a1 2024-11-19T21:40:27,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/9770fc8d584d49069cf20ba08cfe9f15 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/9770fc8d584d49069cf20ba08cfe9f15 2024-11-19T21:40:27,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6afd95c8aba642e1bc3aba2aa9bc7531 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6afd95c8aba642e1bc3aba2aa9bc7531 2024-11-19T21:40:27,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1a1bf19c84c644cf8c3900e34e9879a3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1a1bf19c84c644cf8c3900e34e9879a3 2024-11-19T21:40:27,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/85eca0a4f11b42db94556e2ee337c450 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/85eca0a4f11b42db94556e2ee337c450 2024-11-19T21:40:27,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a4407ac222ca44c78b52eb1e3e41d5c0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a4407ac222ca44c78b52eb1e3e41d5c0 2024-11-19T21:40:27,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/8f6b74996cab4ab4bc9386f5936cc1e3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/8f6b74996cab4ab4bc9386f5936cc1e3 2024-11-19T21:40:27,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/d22994a628cc46df94eb05949c12fbb0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/d22994a628cc46df94eb05949c12fbb0 2024-11-19T21:40:27,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4353d20206d54fb8a736404967ce3a1a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4353d20206d54fb8a736404967ce3a1a 2024-11-19T21:40:27,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/66adef52b81d4ce1ab6bf8c60953aafb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/66adef52b81d4ce1ab6bf8c60953aafb 2024-11-19T21:40:27,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/21f44f787e0348b4bb8b4bba1f8541b0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/21f44f787e0348b4bb8b4bba1f8541b0 2024-11-19T21:40:27,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/aea1d28835114915af2c0d176239ec51 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/aea1d28835114915af2c0d176239ec51 2024-11-19T21:40:27,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4995767f532c4c47b302a7a3d225760a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/4995767f532c4c47b302a7a3d225760a 2024-11-19T21:40:27,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0453ab769c694ab397542fee30a8351a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0453ab769c694ab397542fee30a8351a 2024-11-19T21:40:27,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2067da6f6fd846a39f31f33b4953c070 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2067da6f6fd846a39f31f33b4953c070 2024-11-19T21:40:27,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3a5f27e7c09c4bd3bf8f605bb69f0237 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3a5f27e7c09c4bd3bf8f605bb69f0237 2024-11-19T21:40:27,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b390688f155d4ccda81b9e298a557b5e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b390688f155d4ccda81b9e298a557b5e 2024-11-19T21:40:27,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b61bdcef19f94f86a299380e74646b40 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/b61bdcef19f94f86a299380e74646b40 2024-11-19T21:40:27,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/da094cec2b024764a825ae7bdc304b15 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/da094cec2b024764a825ae7bdc304b15 2024-11-19T21:40:27,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6c35624020234e44bfcb8f62ef43a56d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/6c35624020234e44bfcb8f62ef43a56d 2024-11-19T21:40:27,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/519bd8e07fa749bb9d3c9c9fac4cf16d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/519bd8e07fa749bb9d3c9c9fac4cf16d 2024-11-19T21:40:27,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1641fc28e0aa45af8ec4c0e3f86f93f8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/1641fc28e0aa45af8ec4c0e3f86f93f8 2024-11-19T21:40:27,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/54167cff7d174048b2393fc5147def64 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/54167cff7d174048b2393fc5147def64 2024-11-19T21:40:27,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a263cfce4ad04289bd10be7cba60286f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/a263cfce4ad04289bd10be7cba60286f 2024-11-19T21:40:27,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c77c815d30274a7b9d325c68ed083bf8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c77c815d30274a7b9d325c68ed083bf8 2024-11-19T21:40:27,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/03f51a4653104737a648742838b0b47e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/03f51a4653104737a648742838b0b47e 2024-11-19T21:40:27,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fd25677e6a114d11aeacbe7d09de0943 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/fd25677e6a114d11aeacbe7d09de0943 2024-11-19T21:40:27,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c3e11d42a98d4c8d80f63af5eadc9dd7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/c3e11d42a98d4c8d80f63af5eadc9dd7 2024-11-19T21:40:27,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/acf4cf64280643b5ba30feff7f61e84b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/acf4cf64280643b5ba30feff7f61e84b 2024-11-19T21:40:27,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/5b188e732b52496380a902766fc4d4b2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/5b188e732b52496380a902766fc4d4b2 2024-11-19T21:40:27,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/efd31c3f16dc4392bf17cb81d020c2bf to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/efd31c3f16dc4392bf17cb81d020c2bf 2024-11-19T21:40:27,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2a72b83622ae4b29a6cfc31aa409957c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/2a72b83622ae4b29a6cfc31aa409957c 2024-11-19T21:40:27,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/10a1e843aaed4ab7a4943841ca79a51c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/10a1e843aaed4ab7a4943841ca79a51c 2024-11-19T21:40:27,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d00bd83ba0a42bda1b7ee504d109a28, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/58a01d0e759f4c7f995fd7dd328b63cb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c94118691d2b4bf989c195a9074d3da5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/35a5a06e068e4fb4831cff45715d150c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/27bc37f68f9544729a5e692834e89afb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/dd623138f31a41329bf460f8954743a6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/855f1bee173c4dbe8e3a584a46604ee3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3a7914cdff454831883150258d0545b2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/016a14574d0f4bf088eb75558beb252b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2bb85fd1c6f74de38a59c655e7a898ed, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c0da3e9e757b431e971e8de366c932aa, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8f8315e879654bc3bae560b07b404243, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5ed12c66a40a4ee1a2fbcd94e7f0813f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/cd6fca96feae4347b18a3410398c3521, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/fbb590c9fd1d4c3fbea1ecb4b4a16b69, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0e970455ea8a4f47b7d05b4ba79fdb66, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8267a6a069144b20bf370f298fad5ad3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8040697b878c488fb2f3e1e3ec935178, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/e59b1126470043f89f3a0560ae594342, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/50c8e9ab17a643e3addd573163928bda, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3eff43391c3547889adf4f3da616afb3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/da81f3bde3ae4c88b8646786236ae8e0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bad9a8e72c1a49c7a0425a6f5a3d04e3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2478a8c3827e487685c24ece3973657b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/7c04e98122694b71afee8573671df0d0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bd33b0e40cd9430883e3d129b7f9df86, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/d138afaf3cee4ecaa646da83dbba5cdf, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/48fe066e5d0f4f688cb863d70125db9e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/147b923c04124897b09674b41438fe17, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/99650f727b634e61a46ef9376834c36f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b578f5b038ef458db7ca3610f8008888, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/77f9405d4ab642caab9c7ea1dd792b73, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/1a9dd88135d34e649dd980f4a412d0de, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d1a7a4270814b4c8d69cd81631048bd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5771b706e59c4aa7ab9c7b681f8edf96, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/461345f909414fb6b1f2ff55d8b1f823, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b9bddd56d5d145e2a26054c613427b10, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/187dc428577148a7b8803e0c2c70b853, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0dd9c87520f348be93329405a26fc965, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/deaf790edc034db2b750ac1505f9cb78] to archive 2024-11-19T21:40:27,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:40:27,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d00bd83ba0a42bda1b7ee504d109a28 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d00bd83ba0a42bda1b7ee504d109a28 2024-11-19T21:40:27,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/58a01d0e759f4c7f995fd7dd328b63cb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/58a01d0e759f4c7f995fd7dd328b63cb 2024-11-19T21:40:27,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c94118691d2b4bf989c195a9074d3da5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c94118691d2b4bf989c195a9074d3da5 2024-11-19T21:40:27,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/35a5a06e068e4fb4831cff45715d150c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/35a5a06e068e4fb4831cff45715d150c 2024-11-19T21:40:27,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/27bc37f68f9544729a5e692834e89afb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/27bc37f68f9544729a5e692834e89afb 2024-11-19T21:40:27,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/dd623138f31a41329bf460f8954743a6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/dd623138f31a41329bf460f8954743a6 2024-11-19T21:40:27,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/855f1bee173c4dbe8e3a584a46604ee3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/855f1bee173c4dbe8e3a584a46604ee3 2024-11-19T21:40:27,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3a7914cdff454831883150258d0545b2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3a7914cdff454831883150258d0545b2 2024-11-19T21:40:27,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/016a14574d0f4bf088eb75558beb252b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/016a14574d0f4bf088eb75558beb252b 2024-11-19T21:40:27,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2bb85fd1c6f74de38a59c655e7a898ed to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2bb85fd1c6f74de38a59c655e7a898ed 2024-11-19T21:40:27,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c0da3e9e757b431e971e8de366c932aa to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/c0da3e9e757b431e971e8de366c932aa 2024-11-19T21:40:27,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8f8315e879654bc3bae560b07b404243 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8f8315e879654bc3bae560b07b404243 2024-11-19T21:40:27,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5ed12c66a40a4ee1a2fbcd94e7f0813f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5ed12c66a40a4ee1a2fbcd94e7f0813f 2024-11-19T21:40:27,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/cd6fca96feae4347b18a3410398c3521 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/cd6fca96feae4347b18a3410398c3521 2024-11-19T21:40:27,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/fbb590c9fd1d4c3fbea1ecb4b4a16b69 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/fbb590c9fd1d4c3fbea1ecb4b4a16b69 2024-11-19T21:40:27,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0e970455ea8a4f47b7d05b4ba79fdb66 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0e970455ea8a4f47b7d05b4ba79fdb66 2024-11-19T21:40:27,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8267a6a069144b20bf370f298fad5ad3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8267a6a069144b20bf370f298fad5ad3 2024-11-19T21:40:27,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8040697b878c488fb2f3e1e3ec935178 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/8040697b878c488fb2f3e1e3ec935178 2024-11-19T21:40:27,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/e59b1126470043f89f3a0560ae594342 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/e59b1126470043f89f3a0560ae594342 2024-11-19T21:40:27,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/50c8e9ab17a643e3addd573163928bda to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/50c8e9ab17a643e3addd573163928bda 2024-11-19T21:40:27,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3eff43391c3547889adf4f3da616afb3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3eff43391c3547889adf4f3da616afb3 2024-11-19T21:40:27,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/da81f3bde3ae4c88b8646786236ae8e0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/da81f3bde3ae4c88b8646786236ae8e0 2024-11-19T21:40:27,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bad9a8e72c1a49c7a0425a6f5a3d04e3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bad9a8e72c1a49c7a0425a6f5a3d04e3 2024-11-19T21:40:27,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2478a8c3827e487685c24ece3973657b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/2478a8c3827e487685c24ece3973657b 2024-11-19T21:40:27,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/7c04e98122694b71afee8573671df0d0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/7c04e98122694b71afee8573671df0d0 2024-11-19T21:40:27,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bd33b0e40cd9430883e3d129b7f9df86 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/bd33b0e40cd9430883e3d129b7f9df86 2024-11-19T21:40:27,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/d138afaf3cee4ecaa646da83dbba5cdf to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/d138afaf3cee4ecaa646da83dbba5cdf 2024-11-19T21:40:27,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/48fe066e5d0f4f688cb863d70125db9e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/48fe066e5d0f4f688cb863d70125db9e 2024-11-19T21:40:27,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/147b923c04124897b09674b41438fe17 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/147b923c04124897b09674b41438fe17 2024-11-19T21:40:27,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/99650f727b634e61a46ef9376834c36f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/99650f727b634e61a46ef9376834c36f 2024-11-19T21:40:27,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b578f5b038ef458db7ca3610f8008888 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b578f5b038ef458db7ca3610f8008888 2024-11-19T21:40:27,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/77f9405d4ab642caab9c7ea1dd792b73 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/77f9405d4ab642caab9c7ea1dd792b73 2024-11-19T21:40:27,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/1a9dd88135d34e649dd980f4a412d0de to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/1a9dd88135d34e649dd980f4a412d0de 2024-11-19T21:40:27,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d1a7a4270814b4c8d69cd81631048bd to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/3d1a7a4270814b4c8d69cd81631048bd 2024-11-19T21:40:27,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5771b706e59c4aa7ab9c7b681f8edf96 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/5771b706e59c4aa7ab9c7b681f8edf96 2024-11-19T21:40:27,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/461345f909414fb6b1f2ff55d8b1f823 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/461345f909414fb6b1f2ff55d8b1f823 2024-11-19T21:40:27,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b9bddd56d5d145e2a26054c613427b10 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b9bddd56d5d145e2a26054c613427b10 2024-11-19T21:40:27,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/187dc428577148a7b8803e0c2c70b853 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/187dc428577148a7b8803e0c2c70b853 2024-11-19T21:40:27,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0dd9c87520f348be93329405a26fc965 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/0dd9c87520f348be93329405a26fc965 2024-11-19T21:40:27,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/deaf790edc034db2b750ac1505f9cb78 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/deaf790edc034db2b750ac1505f9cb78 2024-11-19T21:40:27,640 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/recovered.edits/609.seqid, newMaxSeqId=609, maxSeqId=1 2024-11-19T21:40:27,641 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. 2024-11-19T21:40:27,642 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 7f292f8e50ece577fd7cb587fbc66fc6: Waiting for close lock at 1732052426215Running coprocessor pre-close hooks at 1732052426215Disabling compacts and flushes for region at 1732052426215Disabling writes for close at 1732052426215Obtaining lock to block concurrent updates at 1732052426215Preparing flush snapshotting stores in 7f292f8e50ece577fd7cb587fbc66fc6 at 1732052426215Finished memstore snapshotting TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732052426215Flushing stores of TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6. at 1732052426216 (+1 ms)Flushing 7f292f8e50ece577fd7cb587fbc66fc6/A: creating writer at 1732052426216Flushing 7f292f8e50ece577fd7cb587fbc66fc6/A: appending metadata at 1732052426219 (+3 ms)Flushing 7f292f8e50ece577fd7cb587fbc66fc6/A: closing flushed file at 1732052426219Flushing 7f292f8e50ece577fd7cb587fbc66fc6/B: creating writer at 1732052426630 (+411 ms)Flushing 7f292f8e50ece577fd7cb587fbc66fc6/B: appending metadata at 1732052426634 (+4 ms)Flushing 7f292f8e50ece577fd7cb587fbc66fc6/B: closing flushed file at 1732052426634Flushing 7f292f8e50ece577fd7cb587fbc66fc6/C: creating writer at 1732052427046 (+412 ms)Flushing 7f292f8e50ece577fd7cb587fbc66fc6/C: appending metadata at 1732052427049 (+3 ms)Flushing 7f292f8e50ece577fd7cb587fbc66fc6/C: closing flushed file at 1732052427049Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c0c2de6: reopening flushed file at 1732052427463 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ea977f8: reopening flushed file at 1732052427469 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61d5479a: reopening flushed file at 1732052427475 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 7f292f8e50ece577fd7cb587fbc66fc6 in 1267ms, sequenceid=606, compaction requested=true at 1732052427482 (+7 ms)Writing region close event to WAL at 1732052427636 (+154 ms)Running coprocessor post-close hooks at 1732052427641 (+5 ms)Closed at 1732052427641 2024-11-19T21:40:27,645 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:27,645 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7f292f8e50ece577fd7cb587fbc66fc6, regionState=CLOSED 2024-11-19T21:40:27,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:40:27,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T21:40:27,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 7f292f8e50ece577fd7cb587fbc66fc6, server=707262ffa86d,42821,1732052394005 in 1.5900 sec 2024-11-19T21:40:27,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-19T21:40:27,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f292f8e50ece577fd7cb587fbc66fc6, UNASSIGN in 1.5970 sec 2024-11-19T21:40:27,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T21:40:27,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6040 sec 2024-11-19T21:40:27,655 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052427655"}]},"ts":"1732052427655"} 2024-11-19T21:40:27,657 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T21:40:27,657 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T21:40:27,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6210 sec 2024-11-19T21:40:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T21:40:28,188 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:28,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T21:40:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:28,197 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T21:40:28,199 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:28,203 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:28,207 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/recovered.edits] 2024-11-19T21:40:28,211 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/477b2fc2a6dc4c419ab53793b1cbc20a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/477b2fc2a6dc4c419ab53793b1cbc20a 2024-11-19T21:40:28,212 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/98669b7b5e824bfe82b213fc1c4f54c9 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/98669b7b5e824bfe82b213fc1c4f54c9 2024-11-19T21:40:28,214 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/ad8ec27213114d21994ab12cc4fa80d2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/A/ad8ec27213114d21994ab12cc4fa80d2 2024-11-19T21:40:28,216 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0825f876f4624d82b102dc093d1dfb3c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/0825f876f4624d82b102dc093d1dfb3c 2024-11-19T21:40:28,218 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/35a58d0d75f34f27904ffbfa2d53166d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/35a58d0d75f34f27904ffbfa2d53166d 2024-11-19T21:40:28,219 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3e5c75e5f12e475ebeffbcc2f09cd108 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/B/3e5c75e5f12e475ebeffbcc2f09cd108 2024-11-19T21:40:28,222 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b23a8bdd51be4b2ea4b4dde84f923aeb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/b23a8bdd51be4b2ea4b4dde84f923aeb 2024-11-19T21:40:28,224 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/ca13f699fa384c5a831579b087d072ac to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/ca13f699fa384c5a831579b087d072ac 2024-11-19T21:40:28,225 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/f2909afe3a8e40b28a5217f75a4d74ec to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/C/f2909afe3a8e40b28a5217f75a4d74ec 2024-11-19T21:40:28,229 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/recovered.edits/609.seqid to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6/recovered.edits/609.seqid 2024-11-19T21:40:28,229 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/7f292f8e50ece577fd7cb587fbc66fc6 2024-11-19T21:40:28,229 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T21:40:28,235 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:28,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T21:40:28,247 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T21:40:28,283 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T21:40:28,285 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:28,285 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T21:40:28,286 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732052428286"}]},"ts":"9223372036854775807"} 2024-11-19T21:40:28,290 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T21:40:28,291 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7f292f8e50ece577fd7cb587fbc66fc6, NAME => 'TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T21:40:28,291 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T21:40:28,291 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732052428291"}]},"ts":"9223372036854775807"} 2024-11-19T21:40:28,295 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T21:40:28,296 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:28,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-11-19T21:40:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T21:40:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T21:40:28,310 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:28,325 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_349561413_22 at /127.0.0.1:45204 [Waiting for operation #278] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;707262ffa86d:42821-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_525350263_22 at /127.0.0.1:45104 [Waiting for operation #287] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/707262ffa86d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_349561413_22 at /127.0.0.1:34114 [Waiting for operation #248] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=463 (was 305) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5452 (was 5838) 2024-11-19T21:40:28,335 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=463, ProcessCount=11, AvailableMemoryMB=5451 2024-11-19T21:40:28,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T21:40:28,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:40:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:28,340 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T21:40:28,340 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:28,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-19T21:40:28,342 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T21:40:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T21:40:28,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741966_1142 (size=959) 2024-11-19T21:40:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T21:40:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T21:40:28,751 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b 2024-11-19T21:40:28,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741967_1143 (size=53) 2024-11-19T21:40:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T21:40:29,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:40:29,157 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a4602ba7d48e7c270c08c7a6e512b385, disabling compactions & flushes 2024-11-19T21:40:29,157 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:29,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:29,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. after waiting 0 ms 2024-11-19T21:40:29,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:29,158 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:29,158 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a4602ba7d48e7c270c08c7a6e512b385: Waiting for close lock at 1732052429157Disabling compacts and flushes for region at 1732052429157Disabling writes for close at 1732052429158 (+1 ms)Writing region close event to WAL at 1732052429158Closed at 1732052429158 2024-11-19T21:40:29,159 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T21:40:29,160 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732052429159"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732052429159"}]},"ts":"1732052429159"} 2024-11-19T21:40:29,161 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T21:40:29,163 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T21:40:29,163 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052429163"}]},"ts":"1732052429163"} 2024-11-19T21:40:29,164 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T21:40:29,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, ASSIGN}] 2024-11-19T21:40:29,166 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, ASSIGN 2024-11-19T21:40:29,167 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, ASSIGN; state=OFFLINE, location=707262ffa86d,42821,1732052394005; forceNewPlan=false, retain=false 2024-11-19T21:40:29,317 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=a4602ba7d48e7c270c08c7a6e512b385, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:29,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, ASSIGN because future has completed 2024-11-19T21:40:29,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:40:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T21:40:29,482 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:29,482 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:40:29,482 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,482 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:40:29,483 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,483 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,484 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,486 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4602ba7d48e7c270c08c7a6e512b385 columnFamilyName A 2024-11-19T21:40:29,486 DEBUG [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:29,487 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(327): Store=a4602ba7d48e7c270c08c7a6e512b385/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:29,487 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,488 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4602ba7d48e7c270c08c7a6e512b385 columnFamilyName B 2024-11-19T21:40:29,488 DEBUG [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:29,488 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(327): Store=a4602ba7d48e7c270c08c7a6e512b385/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:29,489 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,490 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4602ba7d48e7c270c08c7a6e512b385 columnFamilyName C 2024-11-19T21:40:29,490 DEBUG [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:29,490 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(327): Store=a4602ba7d48e7c270c08c7a6e512b385/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:29,490 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,491 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,491 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,492 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,492 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,493 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:40:29,493 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,495 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:40:29,496 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened a4602ba7d48e7c270c08c7a6e512b385; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73769429, jitterRate=0.09925015270709991}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:40:29,496 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:29,497 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for a4602ba7d48e7c270c08c7a6e512b385: Running coprocessor pre-open hook at 1732052429483Writing region info on filesystem at 1732052429483Initializing all the Stores at 1732052429484 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052429484Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052429484Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052429484Cleaning up temporary data from old regions at 1732052429492 (+8 ms)Running coprocessor post-open hooks at 1732052429496 (+4 ms)Region opened successfully at 1732052429497 (+1 ms) 2024-11-19T21:40:29,498 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., pid=34, masterSystemTime=1732052429472 2024-11-19T21:40:29,501 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:29,501 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:29,501 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=a4602ba7d48e7c270c08c7a6e512b385, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:29,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:40:29,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T21:40:29,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 in 184 msec 2024-11-19T21:40:29,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-19T21:40:29,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, ASSIGN in 342 msec 2024-11-19T21:40:29,510 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T21:40:29,510 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052429510"}]},"ts":"1732052429510"} 2024-11-19T21:40:29,512 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T21:40:29,513 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T21:40:29,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-19T21:40:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T21:40:30,489 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:30,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168ecd40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:30,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:30,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:30,491 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d34e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:30,493 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:30,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab09e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:30,494 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:30,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:30,498 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T21:40:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 707262ffa86d,37541,1732052393169 2024-11-19T21:40:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@340040b4 2024-11-19T21:40:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T21:40:30,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T21:40:30,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T21:40:30,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:40:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:30,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741968_1144 (size=995) 2024-11-19T21:40:30,927 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T21:40:30,927 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T21:40:30,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T21:40:30,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, REOPEN/MOVE}] 2024-11-19T21:40:30,942 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, REOPEN/MOVE 2024-11-19T21:40:30,943 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a4602ba7d48e7c270c08c7a6e512b385, regionState=CLOSING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:30,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, REOPEN/MOVE because future has completed 2024-11-19T21:40:30,946 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T21:40:30,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:40:31,099 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,099 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T21:40:31,099 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing a4602ba7d48e7c270c08c7a6e512b385, disabling compactions & flushes 2024-11-19T21:40:31,099 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,099 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,099 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. after waiting 0 ms 2024-11-19T21:40:31,099 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,104 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T21:40:31,105 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,105 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for a4602ba7d48e7c270c08c7a6e512b385: Waiting for close lock at 1732052431099Running coprocessor pre-close hooks at 1732052431099Disabling compacts and flushes for region at 1732052431099Disabling writes for close at 1732052431099Writing region close event to WAL at 1732052431100 (+1 ms)Running coprocessor post-close hooks at 1732052431105 (+5 ms)Closed at 1732052431105 2024-11-19T21:40:31,105 WARN [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: a4602ba7d48e7c270c08c7a6e512b385 to self. 2024-11-19T21:40:31,112 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,113 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a4602ba7d48e7c270c08c7a6e512b385, regionState=CLOSED 2024-11-19T21:40:31,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:40:31,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-19T21:40:31,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 in 171 msec 2024-11-19T21:40:31,120 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, REOPEN/MOVE; state=CLOSED, location=707262ffa86d,42821,1732052394005; forceNewPlan=false, retain=true 2024-11-19T21:40:31,271 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a4602ba7d48e7c270c08c7a6e512b385, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, REOPEN/MOVE because future has completed 2024-11-19T21:40:31,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:40:31,428 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,428 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:40:31,429 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,429 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:40:31,429 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,429 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,436 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,443 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4602ba7d48e7c270c08c7a6e512b385 columnFamilyName A 2024-11-19T21:40:31,445 DEBUG [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:31,446 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(327): Store=a4602ba7d48e7c270c08c7a6e512b385/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:31,446 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,447 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4602ba7d48e7c270c08c7a6e512b385 columnFamilyName B 2024-11-19T21:40:31,448 DEBUG [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:31,448 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(327): Store=a4602ba7d48e7c270c08c7a6e512b385/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:31,448 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,449 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4602ba7d48e7c270c08c7a6e512b385 columnFamilyName C 2024-11-19T21:40:31,449 DEBUG [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:31,450 INFO [StoreOpener-a4602ba7d48e7c270c08c7a6e512b385-1 {}] regionserver.HStore(327): Store=a4602ba7d48e7c270c08c7a6e512b385/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:31,450 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,451 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,452 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,453 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,453 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,455 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:40:31,457 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,458 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened a4602ba7d48e7c270c08c7a6e512b385; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73167154, jitterRate=0.09027555584907532}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:40:31,458 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,459 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for a4602ba7d48e7c270c08c7a6e512b385: Running coprocessor pre-open hook at 1732052431430Writing region info on filesystem at 1732052431430Initializing all the Stores at 1732052431431 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052431431Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052431434 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052431434Cleaning up temporary data from old regions at 1732052431454 (+20 ms)Running coprocessor post-open hooks at 1732052431458 (+4 ms)Region opened successfully at 1732052431459 (+1 ms) 2024-11-19T21:40:31,460 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., pid=39, masterSystemTime=1732052431424 2024-11-19T21:40:31,463 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,463 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,463 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a4602ba7d48e7c270c08c7a6e512b385, regionState=OPEN, openSeqNum=5, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:40:31,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-19T21:40:31,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 in 193 msec 2024-11-19T21:40:31,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-19T21:40:31,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, REOPEN/MOVE in 530 msec 2024-11-19T21:40:31,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-19T21:40:31,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 546 msec 2024-11-19T21:40:31,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 974 msec 2024-11-19T21:40:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-19T21:40:31,487 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:31,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552fb1a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:31,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:31,489 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c2d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:31,491 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffb4dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f01ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:31,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:31,494 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54499fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:31,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:31,497 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:31,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e930ccf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a283555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:31,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:31,501 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab345f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:31,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:31,503 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f00018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef6e97c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:31,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:31,506 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ea40faa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:31,508 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f50014, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60093174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:31,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:31,511 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed0618e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:31,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:31,513 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391d0419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8203f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:31,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:31,516 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:31,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f79e5ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:31,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:31,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:31,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb98ade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6723353c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:31,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:31,521 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:31,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ee5600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:31,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:31,525 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f28929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aece006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:31,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:31,530 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67aedec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:31,532 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2570f4e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe9245f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,534 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:40:31,534 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:40:31,535 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:40:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:40:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e6f7978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:31,537 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:40:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d54e90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:40:31,539 DEBUG [Thread-720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:31,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:31,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:31,545 DEBUG [Thread-713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:40:31,548 DEBUG [Thread-713 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:31,549 DEBUG [Thread-713 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:31,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:40:31,555 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:31,555 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:31,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:31,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:40:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:40:31,564 DEBUG [Thread-722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:31,566 DEBUG [Thread-722 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:31,567 DEBUG [Thread-722 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:31,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:31,570 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:31,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:40:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:40:31,575 DEBUG [Thread-724 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:31,577 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:31,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:40:31,583 DEBUG [Thread-715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:31,584 DEBUG [Thread-715 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:31,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:31,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:40:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:40:31,590 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:31,591 DEBUG [Thread-718 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:40:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-19T21:40:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:40:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T21:40:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:40:31,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:40:31,595 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:40:31,597 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:31,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052491609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052491608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052491614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052491615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052491619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d1d5bd285fe044d7b6992b3eb329d3a0_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052431562/Put/seqid=0 2024-11-19T21:40:31,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741969_1145 (size=12154) 2024-11-19T21:40:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T21:40:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052491729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052491729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052491739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052491739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052491739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T21:40:31,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:31,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:31,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T21:40:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:31,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:31,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:31,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T21:40:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:31,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052491939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052491939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052491948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052491959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:31,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:31,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052491960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:31,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:31,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T21:40:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:32,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:32,072 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:32,078 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d1d5bd285fe044d7b6992b3eb329d3a0_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d1d5bd285fe044d7b6992b3eb329d3a0_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:32,080 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/10b8995430154f54a69f89a9a7b58580, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:32,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/10b8995430154f54a69f89a9a7b58580 is 175, key is test_row_0/A:col10/1732052431562/Put/seqid=0 2024-11-19T21:40:32,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741970_1146 (size=30955) 2024-11-19T21:40:32,101 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/10b8995430154f54a69f89a9a7b58580 2024-11-19T21:40:32,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/673409d47aa241169538e6c763833a6c is 50, key is test_row_0/B:col10/1732052431562/Put/seqid=0 2024-11-19T21:40:32,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741971_1147 (size=12001) 2024-11-19T21:40:32,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/673409d47aa241169538e6c763833a6c 2024-11-19T21:40:32,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/35debe76cca443afbb5323e3854c2ea5 is 50, key is test_row_0/C:col10/1732052431562/Put/seqid=0 2024-11-19T21:40:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T21:40:32,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T21:40:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:32,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:32,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741972_1148 (size=12001) 2024-11-19T21:40:32,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/35debe76cca443afbb5323e3854c2ea5 2024-11-19T21:40:32,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052492250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052492253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/10b8995430154f54a69f89a9a7b58580 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10b8995430154f54a69f89a9a7b58580 2024-11-19T21:40:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052492259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10b8995430154f54a69f89a9a7b58580, entries=150, sequenceid=16, filesize=30.2 K 2024-11-19T21:40:32,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/673409d47aa241169538e6c763833a6c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/673409d47aa241169538e6c763833a6c 2024-11-19T21:40:32,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052492268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052492269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/673409d47aa241169538e6c763833a6c, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T21:40:32,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/35debe76cca443afbb5323e3854c2ea5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35debe76cca443afbb5323e3854c2ea5 2024-11-19T21:40:32,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35debe76cca443afbb5323e3854c2ea5, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T21:40:32,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a4602ba7d48e7c270c08c7a6e512b385 in 724ms, sequenceid=16, compaction requested=false 2024-11-19T21:40:32,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:32,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T21:40:32,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:32,376 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:40:32,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192c0dee9a34ca4836970a430698760356_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052431615/Put/seqid=0 2024-11-19T21:40:32,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741973_1149 (size=12154) 2024-11-19T21:40:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T21:40:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:32,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:32,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052492786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052492787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052492789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052492789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052492791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:32,857 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192c0dee9a34ca4836970a430698760356_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192c0dee9a34ca4836970a430698760356_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:32,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/30747fb13f6a4bd18bd8bcb88b74cd88, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:32,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/30747fb13f6a4bd18bd8bcb88b74cd88 is 175, key is test_row_0/A:col10/1732052431615/Put/seqid=0 2024-11-19T21:40:32,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741974_1150 (size=30955) 2024-11-19T21:40:32,881 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/30747fb13f6a4bd18bd8bcb88b74cd88 2024-11-19T21:40:32,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/82ab22c475ba4a209bf0309fa153d136 is 50, key is test_row_0/B:col10/1732052431615/Put/seqid=0 2024-11-19T21:40:32,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052492899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052492899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052492899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052492899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:32,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:32,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741975_1151 (size=12001) 2024-11-19T21:40:33,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052493118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052493118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:33,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052493119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052493119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:33,234 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T21:40:33,338 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/82ab22c475ba4a209bf0309fa153d136 2024-11-19T21:40:33,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/478432070f614832b4b9772389227f63 is 50, key is test_row_0/C:col10/1732052431615/Put/seqid=0 2024-11-19T21:40:33,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741976_1152 (size=12001) 2024-11-19T21:40:33,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052493429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:33,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:33,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052493429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052493429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:33,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052493438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T21:40:33,798 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/478432070f614832b4b9772389227f63 2024-11-19T21:40:33,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052493799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/30747fb13f6a4bd18bd8bcb88b74cd88 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/30747fb13f6a4bd18bd8bcb88b74cd88 2024-11-19T21:40:33,807 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:33,814 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/30747fb13f6a4bd18bd8bcb88b74cd88, entries=150, sequenceid=40, filesize=30.2 K 2024-11-19T21:40:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/82ab22c475ba4a209bf0309fa153d136 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/82ab22c475ba4a209bf0309fa153d136 2024-11-19T21:40:33,829 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/82ab22c475ba4a209bf0309fa153d136, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T21:40:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/478432070f614832b4b9772389227f63 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/478432070f614832b4b9772389227f63 2024-11-19T21:40:33,838 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/478432070f614832b4b9772389227f63, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T21:40:33,840 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a4602ba7d48e7c270c08c7a6e512b385 in 1464ms, sequenceid=40, compaction requested=false 2024-11-19T21:40:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-19T21:40:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-19T21:40:33,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T21:40:33,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2450 sec 2024-11-19T21:40:33,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.2550 sec 2024-11-19T21:40:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:33,941 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:33,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:40:33,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190d74ca6fbd3947c2a306f308e3ed0e46_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052432785/Put/seqid=0 2024-11-19T21:40:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052493995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741977_1153 (size=14594) 2024-11-19T21:40:34,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052494007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052494009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052494010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052494118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052494119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052494119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052494119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052494338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052494338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052494338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052494338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,413 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:34,418 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190d74ca6fbd3947c2a306f308e3ed0e46_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d74ca6fbd3947c2a306f308e3ed0e46_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:34,419 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1b79767888ba422bb4f8a045bd0d7bbc, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:34,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1b79767888ba422bb4f8a045bd0d7bbc is 175, key is test_row_0/A:col10/1732052432785/Put/seqid=0 2024-11-19T21:40:34,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741978_1154 (size=39549) 2024-11-19T21:40:34,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052494648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052494648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052494648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052494649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:34,824 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1b79767888ba422bb4f8a045bd0d7bbc 2024-11-19T21:40:34,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/382c012f4e1b4440ae6c84508238bfbc is 50, key is test_row_0/B:col10/1732052432785/Put/seqid=0 2024-11-19T21:40:34,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741979_1155 (size=12001) 2024-11-19T21:40:34,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/382c012f4e1b4440ae6c84508238bfbc 2024-11-19T21:40:34,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/cc7b4ce83c614a18a1f14b8002aa380d is 50, key is test_row_0/C:col10/1732052432785/Put/seqid=0 2024-11-19T21:40:34,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741980_1156 (size=12001) 2024-11-19T21:40:34,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/cc7b4ce83c614a18a1f14b8002aa380d 2024-11-19T21:40:34,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1b79767888ba422bb4f8a045bd0d7bbc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1b79767888ba422bb4f8a045bd0d7bbc 2024-11-19T21:40:34,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1b79767888ba422bb4f8a045bd0d7bbc, entries=200, sequenceid=53, filesize=38.6 K 2024-11-19T21:40:34,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/382c012f4e1b4440ae6c84508238bfbc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/382c012f4e1b4440ae6c84508238bfbc 2024-11-19T21:40:34,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/382c012f4e1b4440ae6c84508238bfbc, entries=150, sequenceid=53, filesize=11.7 K 2024-11-19T21:40:34,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/cc7b4ce83c614a18a1f14b8002aa380d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/cc7b4ce83c614a18a1f14b8002aa380d 2024-11-19T21:40:34,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/cc7b4ce83c614a18a1f14b8002aa380d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-19T21:40:34,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for a4602ba7d48e7c270c08c7a6e512b385 in 949ms, sequenceid=53, compaction requested=true 2024-11-19T21:40:34,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:34,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:34,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:34,891 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:34,891 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:34,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:34,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:34,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:34,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:34,892 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:34,892 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:34,892 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/B is initiating minor compaction (all files) 2024-11-19T21:40:34,892 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/A is initiating minor compaction (all files) 2024-11-19T21:40:34,892 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/B in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:34,892 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/A in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:34,892 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/673409d47aa241169538e6c763833a6c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/82ab22c475ba4a209bf0309fa153d136, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/382c012f4e1b4440ae6c84508238bfbc] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=35.2 K 2024-11-19T21:40:34,892 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10b8995430154f54a69f89a9a7b58580, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/30747fb13f6a4bd18bd8bcb88b74cd88, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1b79767888ba422bb4f8a045bd0d7bbc] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=99.1 K 2024-11-19T21:40:34,892 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:34,892 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10b8995430154f54a69f89a9a7b58580, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/30747fb13f6a4bd18bd8bcb88b74cd88, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1b79767888ba422bb4f8a045bd0d7bbc] 2024-11-19T21:40:34,893 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10b8995430154f54a69f89a9a7b58580, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732052431562 2024-11-19T21:40:34,893 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 673409d47aa241169538e6c763833a6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732052431562 2024-11-19T21:40:34,893 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ab22c475ba4a209bf0309fa153d136, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732052431607 2024-11-19T21:40:34,894 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30747fb13f6a4bd18bd8bcb88b74cd88, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732052431607 2024-11-19T21:40:34,894 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 382c012f4e1b4440ae6c84508238bfbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732052432779 2024-11-19T21:40:34,894 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b79767888ba422bb4f8a045bd0d7bbc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732052432779 2024-11-19T21:40:34,907 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:34,910 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:34,910 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/d2bbf932ea4848b19db096ed67c38cec is 50, key is test_row_0/B:col10/1732052432785/Put/seqid=0 2024-11-19T21:40:34,912 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411197e21a856bb9d4f99b0d71ac52015cf15_a4602ba7d48e7c270c08c7a6e512b385 store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:34,920 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411197e21a856bb9d4f99b0d71ac52015cf15_a4602ba7d48e7c270c08c7a6e512b385, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:34,920 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197e21a856bb9d4f99b0d71ac52015cf15_a4602ba7d48e7c270c08c7a6e512b385 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:34,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741981_1157 (size=12104) 2024-11-19T21:40:34,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741982_1158 (size=4469) 2024-11-19T21:40:34,963 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#A#compaction#138 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:34,964 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/3916a52022cd4b35b7b9525ee3b57ad5 is 175, key is test_row_0/A:col10/1732052432785/Put/seqid=0 2024-11-19T21:40:34,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741983_1159 (size=31058) 2024-11-19T21:40:34,996 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/3916a52022cd4b35b7b9525ee3b57ad5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3916a52022cd4b35b7b9525ee3b57ad5 2024-11-19T21:40:35,005 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/A of a4602ba7d48e7c270c08c7a6e512b385 into 3916a52022cd4b35b7b9525ee3b57ad5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:35,005 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:35,005 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/A, priority=13, startTime=1732052434890; duration=0sec 2024-11-19T21:40:35,005 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:35,005 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:A 2024-11-19T21:40:35,005 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:35,007 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:35,007 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/C is initiating minor compaction (all files) 2024-11-19T21:40:35,007 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/C in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:35,007 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35debe76cca443afbb5323e3854c2ea5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/478432070f614832b4b9772389227f63, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/cc7b4ce83c614a18a1f14b8002aa380d] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=35.2 K 2024-11-19T21:40:35,009 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35debe76cca443afbb5323e3854c2ea5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732052431562 2024-11-19T21:40:35,009 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 478432070f614832b4b9772389227f63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732052431607 2024-11-19T21:40:35,010 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc7b4ce83c614a18a1f14b8002aa380d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732052432779 2024-11-19T21:40:35,033 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:35,034 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/157df3f2d8e4412dbea23e1dc775f1ea is 50, key is test_row_0/C:col10/1732052432785/Put/seqid=0 2024-11-19T21:40:35,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741984_1160 (size=12104) 2024-11-19T21:40:35,163 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:35,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:40:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:35,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196dd421a113f94be18807649398ca8f7e_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052435162/Put/seqid=0 2024-11-19T21:40:35,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741985_1161 (size=12154) 2024-11-19T21:40:35,188 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:35,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052495185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052495185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,193 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196dd421a113f94be18807649398ca8f7e_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196dd421a113f94be18807649398ca8f7e_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:35,195 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/5acea5a031844d478f9d655c42a93107, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:35,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/5acea5a031844d478f9d655c42a93107 is 175, key is test_row_0/A:col10/1732052435162/Put/seqid=0 2024-11-19T21:40:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052495187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052495187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741986_1162 (size=30955) 2024-11-19T21:40:35,203 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/5acea5a031844d478f9d655c42a93107 2024-11-19T21:40:35,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/97e26ca938ab45aab48bb01ba7be86e7 is 50, key is test_row_0/B:col10/1732052435162/Put/seqid=0 2024-11-19T21:40:35,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741987_1163 (size=12001) 2024-11-19T21:40:35,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052495298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052495298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052495308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052495309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,385 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/d2bbf932ea4848b19db096ed67c38cec as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d2bbf932ea4848b19db096ed67c38cec 2024-11-19T21:40:35,392 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/B of a4602ba7d48e7c270c08c7a6e512b385 into d2bbf932ea4848b19db096ed67c38cec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:35,392 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:35,392 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/B, priority=13, startTime=1732052434891; duration=0sec 2024-11-19T21:40:35,392 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:35,393 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:B 2024-11-19T21:40:35,451 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/157df3f2d8e4412dbea23e1dc775f1ea as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/157df3f2d8e4412dbea23e1dc775f1ea 2024-11-19T21:40:35,459 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/C of a4602ba7d48e7c270c08c7a6e512b385 into 157df3f2d8e4412dbea23e1dc775f1ea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:35,459 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:35,459 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/C, priority=13, startTime=1732052434891; duration=0sec 2024-11-19T21:40:35,459 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:35,459 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:C 2024-11-19T21:40:35,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052495508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052495508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052495518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052495518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/97e26ca938ab45aab48bb01ba7be86e7 2024-11-19T21:40:35,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/f3229ff773984b87a2531f96f5e8e75e is 50, key is test_row_0/C:col10/1732052435162/Put/seqid=0 2024-11-19T21:40:35,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741988_1164 (size=12001) 2024-11-19T21:40:35,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/f3229ff773984b87a2531f96f5e8e75e 2024-11-19T21:40:35,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/5acea5a031844d478f9d655c42a93107 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/5acea5a031844d478f9d655c42a93107 2024-11-19T21:40:35,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/5acea5a031844d478f9d655c42a93107, entries=150, sequenceid=80, filesize=30.2 K 2024-11-19T21:40:35,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/97e26ca938ab45aab48bb01ba7be86e7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/97e26ca938ab45aab48bb01ba7be86e7 2024-11-19T21:40:35,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/97e26ca938ab45aab48bb01ba7be86e7, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T21:40:35,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/f3229ff773984b87a2531f96f5e8e75e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f3229ff773984b87a2531f96f5e8e75e 2024-11-19T21:40:35,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f3229ff773984b87a2531f96f5e8e75e, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T21:40:35,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for a4602ba7d48e7c270c08c7a6e512b385 in 530ms, sequenceid=80, compaction requested=false 2024-11-19T21:40:35,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T21:40:35,749 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:35,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T21:40:35,753 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:35,754 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:35,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T21:40:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:35,821 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:35,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:40:35,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a51046418547403e8e99a3c3b3a809b2_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052435819/Put/seqid=0 2024-11-19T21:40:35,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741989_1165 (size=14594) 2024-11-19T21:40:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T21:40:35,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052495888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052495891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052495893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052495893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052495895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:35,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:35,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:35,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:35,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:35,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:35,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:35,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:35,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052495999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052495999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052495999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052495999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052496008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:36,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:36,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T21:40:36,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052496208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052496208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052496208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052496208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:36,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:36,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052496218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,244 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:36,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a51046418547403e8e99a3c3b3a809b2_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a51046418547403e8e99a3c3b3a809b2_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:36,255 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/c9278ad2c5374e3981cb93cb4625e5f9, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:36,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/c9278ad2c5374e3981cb93cb4625e5f9 is 175, key is test_row_0/A:col10/1732052435819/Put/seqid=0 2024-11-19T21:40:36,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741990_1166 (size=39549) 2024-11-19T21:40:36,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:36,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:36,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T21:40:36,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052496519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052496519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052496519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052496519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052496529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:36,670 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/c9278ad2c5374e3981cb93cb4625e5f9 2024-11-19T21:40:36,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/3cf90be655ad481086fe481694d3013a is 50, key is test_row_0/B:col10/1732052435819/Put/seqid=0 2024-11-19T21:40:36,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741991_1167 (size=12001) 2024-11-19T21:40:36,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T21:40:36,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:36,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:36,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:36,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052497028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:37,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052497028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:37,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052497038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052497038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052497039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:37,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/3cf90be655ad481086fe481694d3013a 2024-11-19T21:40:37,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/9f40d56430fa4bdd82ef788d56edbd69 is 50, key is test_row_0/C:col10/1732052435819/Put/seqid=0 2024-11-19T21:40:37,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741992_1168 (size=12001) 2024-11-19T21:40:37,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/9f40d56430fa4bdd82ef788d56edbd69 2024-11-19T21:40:37,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/c9278ad2c5374e3981cb93cb4625e5f9 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/c9278ad2c5374e3981cb93cb4625e5f9 2024-11-19T21:40:37,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/c9278ad2c5374e3981cb93cb4625e5f9, entries=200, sequenceid=94, filesize=38.6 K 2024-11-19T21:40:37,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/3cf90be655ad481086fe481694d3013a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3cf90be655ad481086fe481694d3013a 2024-11-19T21:40:37,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:37,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:37,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3cf90be655ad481086fe481694d3013a, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T21:40:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:37,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/9f40d56430fa4bdd82ef788d56edbd69 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f40d56430fa4bdd82ef788d56edbd69 2024-11-19T21:40:37,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f40d56430fa4bdd82ef788d56edbd69, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T21:40:37,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for a4602ba7d48e7c270c08c7a6e512b385 in 1325ms, sequenceid=94, compaction requested=true 2024-11-19T21:40:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:37,147 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:37,147 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:37,148 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:37,148 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/A is initiating minor compaction (all files) 2024-11-19T21:40:37,149 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/A in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:37,149 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3916a52022cd4b35b7b9525ee3b57ad5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/5acea5a031844d478f9d655c42a93107, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/c9278ad2c5374e3981cb93cb4625e5f9] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=99.2 K 2024-11-19T21:40:37,149 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:37,149 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3916a52022cd4b35b7b9525ee3b57ad5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/5acea5a031844d478f9d655c42a93107, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/c9278ad2c5374e3981cb93cb4625e5f9] 2024-11-19T21:40:37,149 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:37,149 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/B is initiating minor compaction (all files) 2024-11-19T21:40:37,149 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/B in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:37,150 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d2bbf932ea4848b19db096ed67c38cec, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/97e26ca938ab45aab48bb01ba7be86e7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3cf90be655ad481086fe481694d3013a] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=35.3 K 2024-11-19T21:40:37,150 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3916a52022cd4b35b7b9525ee3b57ad5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732052432779 2024-11-19T21:40:37,151 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d2bbf932ea4848b19db096ed67c38cec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732052432779 2024-11-19T21:40:37,151 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5acea5a031844d478f9d655c42a93107, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732052434006 2024-11-19T21:40:37,151 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 97e26ca938ab45aab48bb01ba7be86e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732052434006 2024-11-19T21:40:37,151 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9278ad2c5374e3981cb93cb4625e5f9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052435166 2024-11-19T21:40:37,151 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cf90be655ad481086fe481694d3013a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052435166 2024-11-19T21:40:37,163 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:37,173 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:37,174 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/7174eda8a6ea443fab79a4606bf56581 is 50, key is test_row_0/B:col10/1732052435819/Put/seqid=0 2024-11-19T21:40:37,176 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119c5a5f31e79e74e3f924b5059fa86199c_a4602ba7d48e7c270c08c7a6e512b385 store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:37,183 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119c5a5f31e79e74e3f924b5059fa86199c_a4602ba7d48e7c270c08c7a6e512b385, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:37,183 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c5a5f31e79e74e3f924b5059fa86199c_a4602ba7d48e7c270c08c7a6e512b385 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:37,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741993_1169 (size=12207) 2024-11-19T21:40:37,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741994_1170 (size=4469) 2024-11-19T21:40:37,206 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#A#compaction#147 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:37,207 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/7174eda8a6ea443fab79a4606bf56581 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/7174eda8a6ea443fab79a4606bf56581 2024-11-19T21:40:37,208 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/d01337bc72834d0aa108f3b88ab71dd4 is 175, key is test_row_0/A:col10/1732052435819/Put/seqid=0 2024-11-19T21:40:37,218 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/B of a4602ba7d48e7c270c08c7a6e512b385 into 7174eda8a6ea443fab79a4606bf56581(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:37,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:37,218 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/B, priority=13, startTime=1732052437147; duration=0sec 2024-11-19T21:40:37,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:37,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:B 2024-11-19T21:40:37,219 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:37,220 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:37,220 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/C is initiating minor compaction (all files) 2024-11-19T21:40:37,220 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/C in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:37,220 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/157df3f2d8e4412dbea23e1dc775f1ea, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f3229ff773984b87a2531f96f5e8e75e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f40d56430fa4bdd82ef788d56edbd69] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=35.3 K 2024-11-19T21:40:37,222 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 157df3f2d8e4412dbea23e1dc775f1ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732052432779 2024-11-19T21:40:37,223 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting f3229ff773984b87a2531f96f5e8e75e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732052434006 2024-11-19T21:40:37,223 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f40d56430fa4bdd82ef788d56edbd69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052435166 2024-11-19T21:40:37,241 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:37,241 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/24171bfe5d5047bc87af07acf48e257e is 50, key is test_row_0/C:col10/1732052435819/Put/seqid=0 2024-11-19T21:40:37,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741995_1171 (size=31161) 2024-11-19T21:40:37,256 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/d01337bc72834d0aa108f3b88ab71dd4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/d01337bc72834d0aa108f3b88ab71dd4 2024-11-19T21:40:37,263 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/A of a4602ba7d48e7c270c08c7a6e512b385 into d01337bc72834d0aa108f3b88ab71dd4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:37,263 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:37,263 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/A, priority=13, startTime=1732052437147; duration=0sec 2024-11-19T21:40:37,264 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:37,264 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:A 2024-11-19T21:40:37,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741996_1172 (size=12207) 2024-11-19T21:40:37,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T21:40:37,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:37,292 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T21:40:37,295 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/24171bfe5d5047bc87af07acf48e257e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/24171bfe5d5047bc87af07acf48e257e 2024-11-19T21:40:37,304 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/C of a4602ba7d48e7c270c08c7a6e512b385 into 24171bfe5d5047bc87af07acf48e257e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:37,304 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:37,304 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/C, priority=13, startTime=1732052437147; duration=0sec 2024-11-19T21:40:37,304 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:37,304 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:C 2024-11-19T21:40:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111911501b48d3d94924ad18559f1777dcb5_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052435891/Put/seqid=0 2024-11-19T21:40:37,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741997_1173 (size=12154) 2024-11-19T21:40:37,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,340 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111911501b48d3d94924ad18559f1777dcb5_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111911501b48d3d94924ad18559f1777dcb5_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:37,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/0a47d012a12c45629d53a16e430d1f67, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:37,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/0a47d012a12c45629d53a16e430d1f67 is 175, key is test_row_0/A:col10/1732052435891/Put/seqid=0 2024-11-19T21:40:37,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741998_1174 (size=30955) 2024-11-19T21:40:37,368 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/0a47d012a12c45629d53a16e430d1f67 2024-11-19T21:40:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/d4a1728ead9b472981c74649d5afe5cc is 50, key is test_row_0/B:col10/1732052435891/Put/seqid=0 2024-11-19T21:40:37,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741999_1175 (size=12001) 2024-11-19T21:40:37,404 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/d4a1728ead9b472981c74649d5afe5cc 2024-11-19T21:40:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/2ae0f806835b4eb89968a5dbfef630e8 is 50, key is test_row_0/C:col10/1732052435891/Put/seqid=0 2024-11-19T21:40:37,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742000_1176 (size=12001) 2024-11-19T21:40:37,836 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/2ae0f806835b4eb89968a5dbfef630e8 2024-11-19T21:40:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/0a47d012a12c45629d53a16e430d1f67 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/0a47d012a12c45629d53a16e430d1f67 2024-11-19T21:40:37,845 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/0a47d012a12c45629d53a16e430d1f67, entries=150, sequenceid=122, filesize=30.2 K 2024-11-19T21:40:37,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/d4a1728ead9b472981c74649d5afe5cc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4a1728ead9b472981c74649d5afe5cc 2024-11-19T21:40:37,851 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4a1728ead9b472981c74649d5afe5cc, entries=150, sequenceid=122, filesize=11.7 K 2024-11-19T21:40:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/2ae0f806835b4eb89968a5dbfef630e8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2ae0f806835b4eb89968a5dbfef630e8 2024-11-19T21:40:37,856 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2ae0f806835b4eb89968a5dbfef630e8, entries=150, sequenceid=122, filesize=11.7 K 2024-11-19T21:40:37,857 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for a4602ba7d48e7c270c08c7a6e512b385 in 565ms, sequenceid=122, compaction requested=false 2024-11-19T21:40:37,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:37,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:37,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T21:40:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T21:40:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T21:40:37,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1040 sec 2024-11-19T21:40:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.1130 sec 2024-11-19T21:40:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T21:40:37,909 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T21:40:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,912 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T21:40:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,913 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:37,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T21:40:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T21:40:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,073 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-19T21:40:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111948cc22ee59e7411981f5a02f83101c32_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_2/A:col10/1732052438058/Put/seqid=0 2024-11-19T21:40:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:38,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742001_1177 (size=7274) 2024-11-19T21:40:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,106 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111948cc22ee59e7411981f5a02f83101c32_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111948cc22ee59e7411981f5a02f83101c32_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/43c7b6eb0cae470ca4dc2a93c1730446, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/43c7b6eb0cae470ca4dc2a93c1730446 is 175, key is test_row_2/A:col10/1732052438058/Put/seqid=0 2024-11-19T21:40:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742002_1178 (size=13765) 2024-11-19T21:40:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,128 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/43c7b6eb0cae470ca4dc2a93c1730446 2024-11-19T21:40:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/8c152bca2ea04efbb3a2ab0a89b5617c is 50, key is test_row_2/B:col10/1732052438058/Put/seqid=0 2024-11-19T21:40:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052498152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052498154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052498156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052498158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052498164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:38,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742003_1179 (size=7315) 2024-11-19T21:40:38,182 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/8c152bca2ea04efbb3a2ab0a89b5617c 2024-11-19T21:40:38,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/936066e7a7774e8dabdf0e7a07f66dbd is 50, key is test_row_2/C:col10/1732052438058/Put/seqid=0 2024-11-19T21:40:38,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742004_1180 (size=7315) 2024-11-19T21:40:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T21:40:38,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052498268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052498268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052498268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052498278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052498279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052498478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052498478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052498478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052498488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052498488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T21:40:38,606 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/936066e7a7774e8dabdf0e7a07f66dbd 2024-11-19T21:40:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/43c7b6eb0cae470ca4dc2a93c1730446 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/43c7b6eb0cae470ca4dc2a93c1730446 2024-11-19T21:40:38,620 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/43c7b6eb0cae470ca4dc2a93c1730446, entries=50, sequenceid=127, filesize=13.4 K 2024-11-19T21:40:38,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/8c152bca2ea04efbb3a2ab0a89b5617c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/8c152bca2ea04efbb3a2ab0a89b5617c 2024-11-19T21:40:38,630 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/8c152bca2ea04efbb3a2ab0a89b5617c, entries=50, sequenceid=127, filesize=7.1 K 2024-11-19T21:40:38,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/936066e7a7774e8dabdf0e7a07f66dbd as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/936066e7a7774e8dabdf0e7a07f66dbd 2024-11-19T21:40:38,640 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/936066e7a7774e8dabdf0e7a07f66dbd, entries=50, sequenceid=127, filesize=7.1 K 2024-11-19T21:40:38,641 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=194.56 KB/199230 for a4602ba7d48e7c270c08c7a6e512b385 in 575ms, sequenceid=127, compaction requested=true 2024-11-19T21:40:38,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:38,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:38,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T21:40:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T21:40:38,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T21:40:38,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 730 msec 2024-11-19T21:40:38,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 737 msec 2024-11-19T21:40:38,699 INFO [master/707262ffa86d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T21:40:38,699 INFO [master/707262ffa86d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T21:40:38,792 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:38,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=207.98 KB heapSize=545.67 KB 2024-11-19T21:40:38,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052498789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052498793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052498799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052498795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052498799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fa9628492fb240d5ac7673fb6e313654_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052438161/Put/seqid=0 2024-11-19T21:40:38,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742005_1181 (size=12304) 2024-11-19T21:40:38,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052498909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:38,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052498909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T21:40:39,049 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:39,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T21:40:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T21:40:39,052 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:39,053 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:39,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052499119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052499119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T21:40:39,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T21:40:39,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:39,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,277 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fa9628492fb240d5ac7673fb6e313654_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fa9628492fb240d5ac7673fb6e313654_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:39,278 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/7c55b753444249e7a5c9a5f28af43b5f, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:39,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/7c55b753444249e7a5c9a5f28af43b5f is 175, key is test_row_0/A:col10/1732052438161/Put/seqid=0 2024-11-19T21:40:39,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742006_1182 (size=31105) 2024-11-19T21:40:39,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=69.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/7c55b753444249e7a5c9a5f28af43b5f 2024-11-19T21:40:39,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052499308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052499308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:39,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/5669f7d5246c4af9ae92d58537f8cb35 is 50, key is test_row_0/B:col10/1732052438161/Put/seqid=0 2024-11-19T21:40:39,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052499319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:39,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742007_1183 (size=12151) 2024-11-19T21:40:39,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T21:40:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T21:40:39,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052499428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:39,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052499428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:39,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T21:40:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T21:40:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T21:40:39,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/5669f7d5246c4af9ae92d58537f8cb35 2024-11-19T21:40:39,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/0eab3f873d2540339a0987dfabcb9c7a is 50, key is test_row_0/C:col10/1732052438161/Put/seqid=0 2024-11-19T21:40:39,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742008_1184 (size=12151) 2024-11-19T21:40:39,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/0eab3f873d2540339a0987dfabcb9c7a 2024-11-19T21:40:39,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/7c55b753444249e7a5c9a5f28af43b5f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7c55b753444249e7a5c9a5f28af43b5f 2024-11-19T21:40:39,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7c55b753444249e7a5c9a5f28af43b5f, entries=150, sequenceid=161, filesize=30.4 K 2024-11-19T21:40:39,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/5669f7d5246c4af9ae92d58537f8cb35 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/5669f7d5246c4af9ae92d58537f8cb35 2024-11-19T21:40:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/5669f7d5246c4af9ae92d58537f8cb35, entries=150, sequenceid=161, filesize=11.9 K 2024-11-19T21:40:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/0eab3f873d2540339a0987dfabcb9c7a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/0eab3f873d2540339a0987dfabcb9c7a 2024-11-19T21:40:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/0eab3f873d2540339a0987dfabcb9c7a, entries=150, sequenceid=161, filesize=11.9 K 2024-11-19T21:40:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~207.98 KB/212970, heapSize ~545.63 KB/558720, currentSize=0 B/0 for a4602ba7d48e7c270c08c7a6e512b385 in 1015ms, sequenceid=161, compaction requested=true 2024-11-19T21:40:39,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:39,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:39,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:39,808 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:39,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:39,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:39,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:39,808 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:39,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,809 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43674 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:39,809 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 106986 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:39,809 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/B is initiating minor compaction (all files) 2024-11-19T21:40:39,809 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/A is initiating minor compaction (all files) 2024-11-19T21:40:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,809 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/A in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,809 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/B in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,810 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/d01337bc72834d0aa108f3b88ab71dd4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/0a47d012a12c45629d53a16e430d1f67, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/43c7b6eb0cae470ca4dc2a93c1730446, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7c55b753444249e7a5c9a5f28af43b5f] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=104.5 K 2024-11-19T21:40:39,810 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/7174eda8a6ea443fab79a4606bf56581, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4a1728ead9b472981c74649d5afe5cc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/8c152bca2ea04efbb3a2ab0a89b5617c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/5669f7d5246c4af9ae92d58537f8cb35] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=42.7 K 2024-11-19T21:40:39,810 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,810 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/d01337bc72834d0aa108f3b88ab71dd4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/0a47d012a12c45629d53a16e430d1f67, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/43c7b6eb0cae470ca4dc2a93c1730446, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7c55b753444249e7a5c9a5f28af43b5f] 2024-11-19T21:40:39,810 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 7174eda8a6ea443fab79a4606bf56581, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052435166 2024-11-19T21:40:39,810 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting d01337bc72834d0aa108f3b88ab71dd4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052435166 2024-11-19T21:40:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,810 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d4a1728ead9b472981c74649d5afe5cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732052435889 2024-11-19T21:40:39,810 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a47d012a12c45629d53a16e430d1f67, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732052435889 2024-11-19T21:40:39,811 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c152bca2ea04efbb3a2ab0a89b5617c, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732052438058 2024-11-19T21:40:39,811 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c7b6eb0cae470ca4dc2a93c1730446, keycount=50, bloomtype=ROW, size=13.4 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732052438058 2024-11-19T21:40:39,811 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 5669f7d5246c4af9ae92d58537f8cb35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732052438156 2024-11-19T21:40:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,811 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c55b753444249e7a5c9a5f28af43b5f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732052438156 2024-11-19T21:40:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,822 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:39,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T21:40:39,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:39,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T21:40:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T21:40:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,828 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,828 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/f37bc3f8d227435d9051829f566b8b20 is 50, key is test_row_0/B:col10/1732052438161/Put/seqid=0 2024-11-19T21:40:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T21:40:39,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 774 msec 2024-11-19T21:40:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,833 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411193d54e3458e094dd0975cce10fbec6a31_a4602ba7d48e7c270c08c7a6e512b385 store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 784 msec 2024-11-19T21:40:39,836 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411193d54e3458e094dd0975cce10fbec6a31_a4602ba7d48e7c270c08c7a6e512b385, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,836 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193d54e3458e094dd0975cce10fbec6a31_a4602ba7d48e7c270c08c7a6e512b385 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742009_1185 (size=12493) 2024-11-19T21:40:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742010_1186 (size=4469) 2024-11-19T21:40:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,872 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#A#compaction#159 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,873 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/3be002c27e2a4369aa57a8837e910da6 is 175, key is test_row_0/A:col10/1732052438161/Put/seqid=0 2024-11-19T21:40:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742011_1187 (size=31447) 2024-11-19T21:40:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,888 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/3be002c27e2a4369aa57a8837e910da6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3be002c27e2a4369aa57a8837e910da6 2024-11-19T21:40:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,897 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/A of a4602ba7d48e7c270c08c7a6e512b385 into 3be002c27e2a4369aa57a8837e910da6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:39,897 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:39,897 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/A, priority=12, startTime=1732052439807; duration=0sec 2024-11-19T21:40:39,897 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:39,897 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:A 2024-11-19T21:40:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,897 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,906 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43674 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:39,906 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/C is initiating minor compaction (all files) 2024-11-19T21:40:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,906 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/C in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,906 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/24171bfe5d5047bc87af07acf48e257e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2ae0f806835b4eb89968a5dbfef630e8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/936066e7a7774e8dabdf0e7a07f66dbd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/0eab3f873d2540339a0987dfabcb9c7a] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=42.7 K 2024-11-19T21:40:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,907 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24171bfe5d5047bc87af07acf48e257e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052435166 2024-11-19T21:40:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,908 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ae0f806835b4eb89968a5dbfef630e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732052435889 2024-11-19T21:40:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,909 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936066e7a7774e8dabdf0e7a07f66dbd, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732052438058 2024-11-19T21:40:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,909 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eab3f873d2540339a0987dfabcb9c7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732052438156 2024-11-19T21:40:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,929 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,929 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/2f33c3789313445c9597eb46802660af is 50, key is test_row_0/C:col10/1732052438161/Put/seqid=0 2024-11-19T21:40:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742012_1188 (size=12493) 2024-11-19T21:40:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:40,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:40:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195d68eb991482411c948c5e417477ae0d_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052440030/Put/seqid=0 2024-11-19T21:40:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742013_1189 (size=24758) 2024-11-19T21:40:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,118 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,129 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195d68eb991482411c948c5e417477ae0d_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d68eb991482411c948c5e417477ae0d_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,131 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/21c99c5eaa884a61a6b7f8e8e21f7296, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/21c99c5eaa884a61a6b7f8e8e21f7296 is 175, key is test_row_0/A:col10/1732052440030/Put/seqid=0 2024-11-19T21:40:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742014_1190 (size=74395) 2024-11-19T21:40:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/21c99c5eaa884a61a6b7f8e8e21f7296 2024-11-19T21:40:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/c36037f4800f496694d0e112b9fdfe92 is 50, key is test_row_0/B:col10/1732052440030/Put/seqid=0 2024-11-19T21:40:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742015_1191 (size=24105) 2024-11-19T21:40:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T21:40:40,189 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:40,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T21:40:40,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052500186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T21:40:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,195 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:40,197 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:40,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:40,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052500192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,257 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/f37bc3f8d227435d9051829f566b8b20 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f37bc3f8d227435d9051829f566b8b20 2024-11-19T21:40:40,267 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/B of a4602ba7d48e7c270c08c7a6e512b385 into f37bc3f8d227435d9051829f566b8b20(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:40,267 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:40,267 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/B, priority=12, startTime=1732052439808; duration=0sec 2024-11-19T21:40:40,267 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:40,267 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:B 2024-11-19T21:40:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T21:40:40,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052500299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052500308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052500319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052500319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,320 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:40,321 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052500338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,342 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:40,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T21:40:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,392 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/2f33c3789313445c9597eb46802660af as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2f33c3789313445c9597eb46802660af 2024-11-19T21:40:40,417 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/C of a4602ba7d48e7c270c08c7a6e512b385 into 2f33c3789313445c9597eb46802660af(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:40,417 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:40,417 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/C, priority=12, startTime=1732052439808; duration=0sec 2024-11-19T21:40:40,417 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:40,417 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:C 2024-11-19T21:40:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T21:40:40,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052500508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052500518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T21:40:40,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/c36037f4800f496694d0e112b9fdfe92 2024-11-19T21:40:40,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/bc5f680225b34dda8f32e438ca1ec408 is 50, key is test_row_0/C:col10/1732052440030/Put/seqid=0 2024-11-19T21:40:40,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742016_1192 (size=12151) 2024-11-19T21:40:40,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T21:40:40,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:40,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T21:40:40,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052500818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052500828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:40,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T21:40:40,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:40,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T21:40:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:40,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:41,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/bc5f680225b34dda8f32e438ca1ec408 2024-11-19T21:40:41,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/21c99c5eaa884a61a6b7f8e8e21f7296 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/21c99c5eaa884a61a6b7f8e8e21f7296 2024-11-19T21:40:41,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/21c99c5eaa884a61a6b7f8e8e21f7296, entries=400, sequenceid=174, filesize=72.7 K 2024-11-19T21:40:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/c36037f4800f496694d0e112b9fdfe92 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c36037f4800f496694d0e112b9fdfe92 2024-11-19T21:40:41,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c36037f4800f496694d0e112b9fdfe92, entries=400, sequenceid=174, filesize=23.5 K 2024-11-19T21:40:41,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/bc5f680225b34dda8f32e438ca1ec408 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/bc5f680225b34dda8f32e438ca1ec408 2024-11-19T21:40:41,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/bc5f680225b34dda8f32e438ca1ec408, entries=150, sequenceid=174, filesize=11.9 K 2024-11-19T21:40:41,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a4602ba7d48e7c270c08c7a6e512b385 in 998ms, sequenceid=174, compaction requested=false 2024-11-19T21:40:41,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:41,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T21:40:41,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:41,148 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:40:41,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111912c062dabd764797b06124dbc8b5ac7f_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052440154/Put/seqid=0 2024-11-19T21:40:41,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742017_1193 (size=12304) 2024-11-19T21:40:41,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,211 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111912c062dabd764797b06124dbc8b5ac7f_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111912c062dabd764797b06124dbc8b5ac7f_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:41,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1622b080d7284fedb1b1e07bbe6f5c2a, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:41,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1622b080d7284fedb1b1e07bbe6f5c2a is 175, key is test_row_0/A:col10/1732052440154/Put/seqid=0 2024-11-19T21:40:41,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742018_1194 (size=31105) 2024-11-19T21:40:41,241 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1622b080d7284fedb1b1e07bbe6f5c2a 2024-11-19T21:40:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/b81637017e1f4f208b79997d11bd9c8f is 50, key is test_row_0/B:col10/1732052440154/Put/seqid=0 2024-11-19T21:40:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T21:40:41,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742019_1195 (size=12151) 2024-11-19T21:40:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:41,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:41,334 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/b81637017e1f4f208b79997d11bd9c8f 2024-11-19T21:40:41,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/290e280faf6541d3836df791e0bdb450 is 50, key is test_row_0/C:col10/1732052440154/Put/seqid=0 2024-11-19T21:40:41,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052501371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052501381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:41,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742020_1196 (size=12151) 2024-11-19T21:40:41,426 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/290e280faf6541d3836df791e0bdb450 2024-11-19T21:40:41,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1622b080d7284fedb1b1e07bbe6f5c2a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1622b080d7284fedb1b1e07bbe6f5c2a 2024-11-19T21:40:41,439 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1622b080d7284fedb1b1e07bbe6f5c2a, entries=150, sequenceid=201, filesize=30.4 K 2024-11-19T21:40:41,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/b81637017e1f4f208b79997d11bd9c8f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b81637017e1f4f208b79997d11bd9c8f 2024-11-19T21:40:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,447 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b81637017e1f4f208b79997d11bd9c8f, entries=150, sequenceid=201, filesize=11.9 K 2024-11-19T21:40:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/290e280faf6541d3836df791e0bdb450 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/290e280faf6541d3836df791e0bdb450 2024-11-19T21:40:41,454 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/290e280faf6541d3836df791e0bdb450, entries=150, sequenceid=201, filesize=11.9 K 2024-11-19T21:40:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,455 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a4602ba7d48e7c270c08c7a6e512b385 in 307ms, sequenceid=201, compaction requested=true 2024-11-19T21:40:41,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T21:40:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T21:40:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T21:40:41,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2600 sec 2024-11-19T21:40:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2700 sec 2024-11-19T21:40:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:41,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:40:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e2302ed910cc47efbaf1bc529e5da374_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052441516/Put/seqid=0 2024-11-19T21:40:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742021_1197 (size=14794) 2024-11-19T21:40:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052501622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:41,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052501630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:41,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052501738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:41,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052501748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:41,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052501958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:41,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052501959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:41,989 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:41,993 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e2302ed910cc47efbaf1bc529e5da374_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e2302ed910cc47efbaf1bc529e5da374_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:41,999 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/aeec80f101c94abf92c059dcd02b2d01, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:42,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/aeec80f101c94abf92c059dcd02b2d01 is 175, key is test_row_0/A:col10/1732052441516/Put/seqid=0 2024-11-19T21:40:42,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742022_1198 (size=39749) 2024-11-19T21:40:42,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/aeec80f101c94abf92c059dcd02b2d01 2024-11-19T21:40:42,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/c30936cbb59141bb831c1f376b2aa179 is 50, key is test_row_0/B:col10/1732052441516/Put/seqid=0 2024-11-19T21:40:42,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742023_1199 (size=12151) 2024-11-19T21:40:42,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/c30936cbb59141bb831c1f376b2aa179 2024-11-19T21:40:42,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/4437df20a0494d42b21095cb58533fbf is 50, key is test_row_0/C:col10/1732052441516/Put/seqid=0 2024-11-19T21:40:42,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742024_1200 (size=12151) 2024-11-19T21:40:42,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:42,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052502268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:42,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052502268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T21:40:42,339 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:42,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:42,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052502338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:42,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052502338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:42,340 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:42,342 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T21:40:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T21:40:42,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:42,347 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:42,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052502368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:42,372 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T21:40:42,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T21:40:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:42,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:42,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:42,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/4437df20a0494d42b21095cb58533fbf 2024-11-19T21:40:42,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/aeec80f101c94abf92c059dcd02b2d01 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aeec80f101c94abf92c059dcd02b2d01 2024-11-19T21:40:42,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aeec80f101c94abf92c059dcd02b2d01, entries=200, sequenceid=214, filesize=38.8 K 2024-11-19T21:40:42,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/c30936cbb59141bb831c1f376b2aa179 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c30936cbb59141bb831c1f376b2aa179 2024-11-19T21:40:42,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c30936cbb59141bb831c1f376b2aa179, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T21:40:42,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/4437df20a0494d42b21095cb58533fbf as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/4437df20a0494d42b21095cb58533fbf 2024-11-19T21:40:42,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/4437df20a0494d42b21095cb58533fbf, entries=150, sequenceid=214, filesize=11.9 K 2024-11-19T21:40:42,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a4602ba7d48e7c270c08c7a6e512b385 in 1072ms, sequenceid=214, compaction requested=true 2024-11-19T21:40:42,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:42,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:42,589 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:42,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:42,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:42,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:42,589 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:42,590 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176696 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:42,590 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/A is initiating minor compaction (all files) 2024-11-19T21:40:42,590 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/A in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:42,591 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3be002c27e2a4369aa57a8837e910da6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/21c99c5eaa884a61a6b7f8e8e21f7296, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1622b080d7284fedb1b1e07bbe6f5c2a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aeec80f101c94abf92c059dcd02b2d01] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=172.6 K 2024-11-19T21:40:42,591 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:42,591 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3be002c27e2a4369aa57a8837e910da6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/21c99c5eaa884a61a6b7f8e8e21f7296, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1622b080d7284fedb1b1e07bbe6f5c2a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aeec80f101c94abf92c059dcd02b2d01] 2024-11-19T21:40:42,591 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60900 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:42,591 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/B is initiating minor compaction (all files) 2024-11-19T21:40:42,591 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/B in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:42,591 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f37bc3f8d227435d9051829f566b8b20, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c36037f4800f496694d0e112b9fdfe92, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b81637017e1f4f208b79997d11bd9c8f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c30936cbb59141bb831c1f376b2aa179] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=59.5 K 2024-11-19T21:40:42,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:42,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:42,592 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting f37bc3f8d227435d9051829f566b8b20, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732052438156 2024-11-19T21:40:42,592 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3be002c27e2a4369aa57a8837e910da6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732052438156 2024-11-19T21:40:42,592 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21c99c5eaa884a61a6b7f8e8e21f7296, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732052439955 2024-11-19T21:40:42,592 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c36037f4800f496694d0e112b9fdfe92, keycount=400, bloomtype=ROW, size=23.5 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732052439955 2024-11-19T21:40:42,592 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1622b080d7284fedb1b1e07bbe6f5c2a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732052440154 2024-11-19T21:40:42,592 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b81637017e1f4f208b79997d11bd9c8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732052440154 2024-11-19T21:40:42,593 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c30936cbb59141bb831c1f376b2aa179, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732052441362 2024-11-19T21:40:42,593 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeec80f101c94abf92c059dcd02b2d01, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732052441362 2024-11-19T21:40:42,611 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:42,615 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:42,616 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/d4f65a9825fc4afcbddd67d0a2a42c9c is 50, key is test_row_0/B:col10/1732052441516/Put/seqid=0 2024-11-19T21:40:42,630 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111929a69de1cfab47b485a9d882f0ceebe7_a4602ba7d48e7c270c08c7a6e512b385 store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:42,634 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111929a69de1cfab47b485a9d882f0ceebe7_a4602ba7d48e7c270c08c7a6e512b385, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:42,634 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111929a69de1cfab47b485a9d882f0ceebe7_a4602ba7d48e7c270c08c7a6e512b385 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:42,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T21:40:42,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:42,655 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:40:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T21:40:42,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742025_1201 (size=12629) 2024-11-19T21:40:42,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742026_1202 (size=4469) 2024-11-19T21:40:42,681 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#A#compaction#171 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:42,682 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fbe5129e4e314de09d46c3fe4fdc0c78 is 175, key is test_row_0/A:col10/1732052441516/Put/seqid=0 2024-11-19T21:40:42,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d2b8a1705bd84766b695c729ff7fe04f_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052441628/Put/seqid=0 2024-11-19T21:40:42,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742027_1203 (size=31583) 2024-11-19T21:40:42,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742028_1204 (size=12304) 2024-11-19T21:40:42,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:42,774 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fbe5129e4e314de09d46c3fe4fdc0c78 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe5129e4e314de09d46c3fe4fdc0c78 2024-11-19T21:40:42,775 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d2b8a1705bd84766b695c729ff7fe04f_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d2b8a1705bd84766b695c729ff7fe04f_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:42,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/327b771195194c82b34d0cdb5489f941, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:42,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/327b771195194c82b34d0cdb5489f941 is 175, key is test_row_0/A:col10/1732052441628/Put/seqid=0 2024-11-19T21:40:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:42,790 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/A of a4602ba7d48e7c270c08c7a6e512b385 into fbe5129e4e314de09d46c3fe4fdc0c78(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:42,790 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:42,790 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/A, priority=12, startTime=1732052442589; duration=0sec 2024-11-19T21:40:42,790 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:42,790 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:A 2024-11-19T21:40:42,790 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:42,792 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:42,792 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/C is initiating minor compaction (all files) 2024-11-19T21:40:42,792 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/C in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:42,792 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2f33c3789313445c9597eb46802660af, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/bc5f680225b34dda8f32e438ca1ec408, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/290e280faf6541d3836df791e0bdb450, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/4437df20a0494d42b21095cb58533fbf] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=47.8 K 2024-11-19T21:40:42,792 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f33c3789313445c9597eb46802660af, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732052438156 2024-11-19T21:40:42,796 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc5f680225b34dda8f32e438ca1ec408, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732052439957 2024-11-19T21:40:42,797 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 290e280faf6541d3836df791e0bdb450, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732052440154 2024-11-19T21:40:42,798 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4437df20a0494d42b21095cb58533fbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732052441362 2024-11-19T21:40:42,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742029_1205 (size=31105) 2024-11-19T21:40:42,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052502825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:42,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052502827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:42,851 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:42,852 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/35d26be9fef2490bb3e706781888620f is 50, key is test_row_0/C:col10/1732052441516/Put/seqid=0 2024-11-19T21:40:42,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742030_1206 (size=12629) 2024-11-19T21:40:42,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052502948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:42,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052502948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T21:40:43,081 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/d4f65a9825fc4afcbddd67d0a2a42c9c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4f65a9825fc4afcbddd67d0a2a42c9c 2024-11-19T21:40:43,086 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/B of a4602ba7d48e7c270c08c7a6e512b385 into d4f65a9825fc4afcbddd67d0a2a42c9c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:43,086 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:43,086 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/B, priority=12, startTime=1732052442589; duration=0sec 2024-11-19T21:40:43,086 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:43,086 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:B 2024-11-19T21:40:43,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052503158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:43,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052503158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:43,226 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/327b771195194c82b34d0cdb5489f941 2024-11-19T21:40:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/6e92d8a899ff417fbdeb1777030e2c13 is 50, key is test_row_0/B:col10/1732052441628/Put/seqid=0 2024-11-19T21:40:43,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742031_1207 (size=12151) 2024-11-19T21:40:43,278 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/35d26be9fef2490bb3e706781888620f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35d26be9fef2490bb3e706781888620f 2024-11-19T21:40:43,283 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/C of a4602ba7d48e7c270c08c7a6e512b385 into 35d26be9fef2490bb3e706781888620f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:43,283 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:43,283 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/C, priority=12, startTime=1732052442589; duration=0sec 2024-11-19T21:40:43,284 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:43,284 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:C 2024-11-19T21:40:43,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052503468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:43,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052503468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T21:40:43,651 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/6e92d8a899ff417fbdeb1777030e2c13 2024-11-19T21:40:43,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/3f2b906fdf234200b27841104023acc7 is 50, key is test_row_0/C:col10/1732052441628/Put/seqid=0 2024-11-19T21:40:43,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742032_1208 (size=12151) 2024-11-19T21:40:43,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052503978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052503978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:44,066 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/3f2b906fdf234200b27841104023acc7 2024-11-19T21:40:44,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/327b771195194c82b34d0cdb5489f941 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/327b771195194c82b34d0cdb5489f941 2024-11-19T21:40:44,074 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/327b771195194c82b34d0cdb5489f941, entries=150, sequenceid=237, filesize=30.4 K 2024-11-19T21:40:44,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/6e92d8a899ff417fbdeb1777030e2c13 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/6e92d8a899ff417fbdeb1777030e2c13 2024-11-19T21:40:44,079 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/6e92d8a899ff417fbdeb1777030e2c13, entries=150, sequenceid=237, filesize=11.9 K 2024-11-19T21:40:44,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/3f2b906fdf234200b27841104023acc7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3f2b906fdf234200b27841104023acc7 2024-11-19T21:40:44,084 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3f2b906fdf234200b27841104023acc7, entries=150, sequenceid=237, filesize=11.9 K 2024-11-19T21:40:44,086 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for a4602ba7d48e7c270c08c7a6e512b385 in 1431ms, sequenceid=237, compaction requested=false 2024-11-19T21:40:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T21:40:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T21:40:44,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T21:40:44,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-11-19T21:40:44,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.7510 sec 2024-11-19T21:40:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T21:40:44,488 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:44,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T21:40:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T21:40:44,490 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:44,491 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:44,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T21:40:44,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T21:40:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:44,643 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T21:40:44,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190ab120e5c48a42c3b77b27dec1da6772_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052442822/Put/seqid=0 2024-11-19T21:40:44,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742033_1209 (size=12304) 2024-11-19T21:40:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T21:40:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:44,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:45,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052505022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:45,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052505025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:45,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:45,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:45,057 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190ab120e5c48a42c3b77b27dec1da6772_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190ab120e5c48a42c3b77b27dec1da6772_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/598fcbfaadd34a06b6e7b19744e1e8b4, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/598fcbfaadd34a06b6e7b19744e1e8b4 is 175, key is test_row_0/A:col10/1732052442822/Put/seqid=0 2024-11-19T21:40:45,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742034_1210 (size=31105) 2024-11-19T21:40:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T21:40:45,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052505138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:45,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052505138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:45,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:45,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052505348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052505348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:45,464 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/598fcbfaadd34a06b6e7b19744e1e8b4 2024-11-19T21:40:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/b925d723b79d4dcab41a47b8e7933889 is 50, key is test_row_0/B:col10/1732052442822/Put/seqid=0 2024-11-19T21:40:45,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742035_1211 (size=12151) 2024-11-19T21:40:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T21:40:45,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052505658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:45,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052505658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:45,885 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/b925d723b79d4dcab41a47b8e7933889 2024-11-19T21:40:45,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/6c2efb9fa35e449cb917ff87f028449b is 50, key is test_row_0/C:col10/1732052442822/Put/seqid=0 2024-11-19T21:40:45,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742036_1212 (size=12151) 2024-11-19T21:40:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052506168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052506168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:46,296 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/6c2efb9fa35e449cb917ff87f028449b 2024-11-19T21:40:46,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/598fcbfaadd34a06b6e7b19744e1e8b4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/598fcbfaadd34a06b6e7b19744e1e8b4 2024-11-19T21:40:46,304 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/598fcbfaadd34a06b6e7b19744e1e8b4, entries=150, sequenceid=254, filesize=30.4 K 2024-11-19T21:40:46,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/b925d723b79d4dcab41a47b8e7933889 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b925d723b79d4dcab41a47b8e7933889 2024-11-19T21:40:46,310 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b925d723b79d4dcab41a47b8e7933889, entries=150, sequenceid=254, filesize=11.9 K 2024-11-19T21:40:46,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/6c2efb9fa35e449cb917ff87f028449b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6c2efb9fa35e449cb917ff87f028449b 2024-11-19T21:40:46,316 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6c2efb9fa35e449cb917ff87f028449b, entries=150, sequenceid=254, filesize=11.9 K 2024-11-19T21:40:46,317 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a4602ba7d48e7c270c08c7a6e512b385 in 1674ms, sequenceid=254, compaction requested=true 2024-11-19T21:40:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T21:40:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T21:40:46,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T21:40:46,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-11-19T21:40:46,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8320 sec 2024-11-19T21:40:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:46,370 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:46,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:40:46,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111922a821cdf6d147229713cae217314017_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052445020/Put/seqid=0 2024-11-19T21:40:46,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742037_1213 (size=14994) 2024-11-19T21:40:46,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:46,386 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111922a821cdf6d147229713cae217314017_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111922a821cdf6d147229713cae217314017_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:46,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/af5f63dce63e4684961b911a561bdc15, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:46,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/af5f63dce63e4684961b911a561bdc15 is 175, key is test_row_0/A:col10/1732052445020/Put/seqid=0 2024-11-19T21:40:46,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742038_1214 (size=39949) 2024-11-19T21:40:46,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052506394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:46,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052506394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:46,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42198 deadline: 1732052506418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:46,421 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:46,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052506508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:46,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052506508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T21:40:46,642 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:46,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-19T21:40:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T21:40:46,644 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:46,644 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:46,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:46,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052506718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:46,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052506718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T21:40:46,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/af5f63dce63e4684961b911a561bdc15 2024-11-19T21:40:46,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T21:40:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:46,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:46,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/510d894c530041c981411191562fb627 is 50, key is test_row_0/B:col10/1732052445020/Put/seqid=0 2024-11-19T21:40:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742039_1215 (size=12301) 2024-11-19T21:40:46,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T21:40:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:46,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:46,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T21:40:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052507028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052507028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:47,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T21:40:47,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:47,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:47,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:47,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:47,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:47,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052507178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,181 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:47,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052507188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,192 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:47,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/510d894c530041c981411191562fb627 2024-11-19T21:40:47,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/38aa6b2e99204e0bb643f4140ee44a8b is 50, key is test_row_0/C:col10/1732052445020/Put/seqid=0 2024-11-19T21:40:47,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742040_1216 (size=12301) 2024-11-19T21:40:47,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/38aa6b2e99204e0bb643f4140ee44a8b 2024-11-19T21:40:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/af5f63dce63e4684961b911a561bdc15 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/af5f63dce63e4684961b911a561bdc15 2024-11-19T21:40:47,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/af5f63dce63e4684961b911a561bdc15, entries=200, sequenceid=277, filesize=39.0 K 2024-11-19T21:40:47,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/510d894c530041c981411191562fb627 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/510d894c530041c981411191562fb627 2024-11-19T21:40:47,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/510d894c530041c981411191562fb627, entries=150, sequenceid=277, filesize=12.0 K 2024-11-19T21:40:47,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/38aa6b2e99204e0bb643f4140ee44a8b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/38aa6b2e99204e0bb643f4140ee44a8b 2024-11-19T21:40:47,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/38aa6b2e99204e0bb643f4140ee44a8b, entries=150, sequenceid=277, filesize=12.0 K 2024-11-19T21:40:47,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a4602ba7d48e7c270c08c7a6e512b385 in 871ms, sequenceid=277, compaction requested=true 2024-11-19T21:40:47,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:47,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:47,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:47,241 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:47,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:47,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:47,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:47,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T21:40:47,241 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:47,247 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:47,247 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/B is initiating minor compaction (all files) 2024-11-19T21:40:47,247 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/B in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:47,248 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4f65a9825fc4afcbddd67d0a2a42c9c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/6e92d8a899ff417fbdeb1777030e2c13, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b925d723b79d4dcab41a47b8e7933889, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/510d894c530041c981411191562fb627] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=48.1 K 2024-11-19T21:40:47,248 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:47,248 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/A is initiating minor compaction (all files) 2024-11-19T21:40:47,248 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/A in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:47,248 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe5129e4e314de09d46c3fe4fdc0c78, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/327b771195194c82b34d0cdb5489f941, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/598fcbfaadd34a06b6e7b19744e1e8b4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/af5f63dce63e4684961b911a561bdc15] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=130.6 K 2024-11-19T21:40:47,248 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:47,248 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe5129e4e314de09d46c3fe4fdc0c78, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/327b771195194c82b34d0cdb5489f941, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/598fcbfaadd34a06b6e7b19744e1e8b4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/af5f63dce63e4684961b911a561bdc15] 2024-11-19T21:40:47,249 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe5129e4e314de09d46c3fe4fdc0c78, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732052441362 2024-11-19T21:40:47,249 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d4f65a9825fc4afcbddd67d0a2a42c9c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732052441362 2024-11-19T21:40:47,249 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e92d8a899ff417fbdeb1777030e2c13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732052441589 2024-11-19T21:40:47,249 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 327b771195194c82b34d0cdb5489f941, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732052441589 2024-11-19T21:40:47,251 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b925d723b79d4dcab41a47b8e7933889, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732052442783 2024-11-19T21:40:47,251 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598fcbfaadd34a06b6e7b19744e1e8b4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732052442783 2024-11-19T21:40:47,251 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 510d894c530041c981411191562fb627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732052445020 2024-11-19T21:40:47,251 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting af5f63dce63e4684961b911a561bdc15, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732052445014 2024-11-19T21:40:47,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T21:40:47,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:47,256 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:40:47,266 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#B#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:47,266 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/90a34601b6c7458db9fd62fde5aaab03 is 50, key is test_row_0/B:col10/1732052445020/Put/seqid=0 2024-11-19T21:40:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T21:40:47,271 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d200a981de584ae281d125592a3f42fc_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052446389/Put/seqid=0 2024-11-19T21:40:47,279 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111994128b86706741ddbe314a8c2fd2c781_a4602ba7d48e7c270c08c7a6e512b385 store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:47,282 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111994128b86706741ddbe314a8c2fd2c781_a4602ba7d48e7c270c08c7a6e512b385, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:47,282 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111994128b86706741ddbe314a8c2fd2c781_a4602ba7d48e7c270c08c7a6e512b385 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:47,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742041_1217 (size=12915) 2024-11-19T21:40:47,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742043_1219 (size=4469) 2024-11-19T21:40:47,298 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#A#compaction#184 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:47,299 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/ba8f857e786b4d67bbeb0c831b2ea1a0 is 175, key is test_row_0/A:col10/1732052445020/Put/seqid=0 2024-11-19T21:40:47,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742042_1218 (size=12454) 2024-11-19T21:40:47,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:47,327 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d200a981de584ae281d125592a3f42fc_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d200a981de584ae281d125592a3f42fc_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:47,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/7af7080d9db74c26912da486ead46670, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:47,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/7af7080d9db74c26912da486ead46670 is 175, key is test_row_0/A:col10/1732052446389/Put/seqid=0 2024-11-19T21:40:47,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742044_1220 (size=31869) 2024-11-19T21:40:47,356 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/ba8f857e786b4d67bbeb0c831b2ea1a0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/ba8f857e786b4d67bbeb0c831b2ea1a0 2024-11-19T21:40:47,363 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/A of a4602ba7d48e7c270c08c7a6e512b385 into ba8f857e786b4d67bbeb0c831b2ea1a0(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:47,363 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:47,363 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/A, priority=12, startTime=1732052447241; duration=0sec 2024-11-19T21:40:47,363 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:47,363 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:A 2024-11-19T21:40:47,363 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:40:47,367 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:40:47,367 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/C is initiating minor compaction (all files) 2024-11-19T21:40:47,367 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/C in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:47,368 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35d26be9fef2490bb3e706781888620f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3f2b906fdf234200b27841104023acc7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6c2efb9fa35e449cb917ff87f028449b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/38aa6b2e99204e0bb643f4140ee44a8b] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=48.1 K 2024-11-19T21:40:47,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742045_1221 (size=31255) 2024-11-19T21:40:47,371 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35d26be9fef2490bb3e706781888620f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732052441362 2024-11-19T21:40:47,371 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f2b906fdf234200b27841104023acc7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732052441589 2024-11-19T21:40:47,372 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c2efb9fa35e449cb917ff87f028449b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732052442783 2024-11-19T21:40:47,372 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38aa6b2e99204e0bb643f4140ee44a8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732052445020 2024-11-19T21:40:47,385 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#C#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:47,386 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/9f4a7172f9574793883f35fe66e55a3a is 50, key is test_row_0/C:col10/1732052445020/Put/seqid=0 2024-11-19T21:40:47,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742046_1222 (size=12915) 2024-11-19T21:40:47,409 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/9f4a7172f9574793883f35fe66e55a3a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f4a7172f9574793883f35fe66e55a3a 2024-11-19T21:40:47,415 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/C of a4602ba7d48e7c270c08c7a6e512b385 into 9f4a7172f9574793883f35fe66e55a3a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:47,416 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:47,416 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/C, priority=12, startTime=1732052447241; duration=0sec 2024-11-19T21:40:47,416 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:47,416 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:C 2024-11-19T21:40:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:47,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052507625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052507628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:47,691 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/90a34601b6c7458db9fd62fde5aaab03 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/90a34601b6c7458db9fd62fde5aaab03 2024-11-19T21:40:47,701 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/B of a4602ba7d48e7c270c08c7a6e512b385 into 90a34601b6c7458db9fd62fde5aaab03(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:47,701 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:47,702 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/B, priority=12, startTime=1732052447241; duration=0sec 2024-11-19T21:40:47,702 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:47,702 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:B 2024-11-19T21:40:47,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052507749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052507749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:47,772 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/7af7080d9db74c26912da486ead46670 2024-11-19T21:40:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T21:40:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/764cc11bfe884e47865f176eb8306b9c is 50, key is test_row_0/B:col10/1732052446389/Put/seqid=0 2024-11-19T21:40:47,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742047_1223 (size=12301) 2024-11-19T21:40:47,807 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/764cc11bfe884e47865f176eb8306b9c 2024-11-19T21:40:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/b13d30529cfa4b1ba4b2f8cace3ead8e is 50, key is test_row_0/C:col10/1732052446389/Put/seqid=0 2024-11-19T21:40:47,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742048_1224 (size=12301) 2024-11-19T21:40:47,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052507958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052507958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:48,243 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/b13d30529cfa4b1ba4b2f8cace3ead8e 2024-11-19T21:40:48,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/7af7080d9db74c26912da486ead46670 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7af7080d9db74c26912da486ead46670 2024-11-19T21:40:48,254 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7af7080d9db74c26912da486ead46670, entries=150, sequenceid=290, filesize=30.5 K 2024-11-19T21:40:48,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/764cc11bfe884e47865f176eb8306b9c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/764cc11bfe884e47865f176eb8306b9c 2024-11-19T21:40:48,264 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/764cc11bfe884e47865f176eb8306b9c, entries=150, sequenceid=290, filesize=12.0 K 2024-11-19T21:40:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/b13d30529cfa4b1ba4b2f8cace3ead8e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/b13d30529cfa4b1ba4b2f8cace3ead8e 2024-11-19T21:40:48,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052508269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:48,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052508270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:48,274 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/b13d30529cfa4b1ba4b2f8cace3ead8e, entries=150, sequenceid=290, filesize=12.0 K 2024-11-19T21:40:48,275 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for a4602ba7d48e7c270c08c7a6e512b385 in 1019ms, sequenceid=290, compaction requested=false 2024-11-19T21:40:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-19T21:40:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-19T21:40:48,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T21:40:48,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6330 sec 2024-11-19T21:40:48,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.6380 sec 2024-11-19T21:40:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:48,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:48,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:40:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T21:40:48,788 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:48,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:40:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-19T21:40:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T21:40:48,791 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:40:48,791 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:40:48,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:40:48,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f6f9a7ec75824b10af61aa8ef33dd211_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052447618/Put/seqid=0 2024-11-19T21:40:48,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052508804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:48,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052508808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:48,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742049_1225 (size=14994) 2024-11-19T21:40:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T21:40:48,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052508918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:48,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052508918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:48,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:48,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:48,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T21:40:49,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:49,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:49,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052509139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052509139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42184 deadline: 1732052509188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,195 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42126 deadline: 1732052509218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,221 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in a4602ba7d48e7c270c08c7a6e512b385 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:40:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,226 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:49,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f6f9a7ec75824b10af61aa8ef33dd211_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f6f9a7ec75824b10af61aa8ef33dd211_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:49,235 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1e43d40525494198835abf0e98ec6f15, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:49,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1e43d40525494198835abf0e98ec6f15 is 175, key is test_row_0/A:col10/1732052447618/Put/seqid=0 2024-11-19T21:40:49,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742050_1226 (size=39949) 2024-11-19T21:40:49,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:49,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1e43d40525494198835abf0e98ec6f15 2024-11-19T21:40:49,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:49,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/3ad2c96e84e54e53a1716ccf7a8b4804 is 50, key is test_row_0/B:col10/1732052447618/Put/seqid=0 2024-11-19T21:40:49,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742051_1227 (size=12301) 2024-11-19T21:40:49,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/3ad2c96e84e54e53a1716ccf7a8b4804 2024-11-19T21:40:49,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/ef3dc3d01b0142ec9bedfba68795066d is 50, key is test_row_0/C:col10/1732052447618/Put/seqid=0 2024-11-19T21:40:49,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742052_1228 (size=12301) 2024-11-19T21:40:49,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/ef3dc3d01b0142ec9bedfba68795066d 2024-11-19T21:40:49,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/1e43d40525494198835abf0e98ec6f15 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1e43d40525494198835abf0e98ec6f15 2024-11-19T21:40:49,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1e43d40525494198835abf0e98ec6f15, entries=200, sequenceid=318, filesize=39.0 K 2024-11-19T21:40:49,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/3ad2c96e84e54e53a1716ccf7a8b4804 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3ad2c96e84e54e53a1716ccf7a8b4804 2024-11-19T21:40:49,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3ad2c96e84e54e53a1716ccf7a8b4804, entries=150, sequenceid=318, filesize=12.0 K 2024-11-19T21:40:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T21:40:49,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:49,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:49,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/ef3dc3d01b0142ec9bedfba68795066d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ef3dc3d01b0142ec9bedfba68795066d 2024-11-19T21:40:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ef3dc3d01b0142ec9bedfba68795066d, entries=150, sequenceid=318, filesize=12.0 K 2024-11-19T21:40:49,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a4602ba7d48e7c270c08c7a6e512b385 in 651ms, sequenceid=318, compaction requested=true 2024-11-19T21:40:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:49,433 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:49,433 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:49,435 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:49,435 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/B is initiating minor compaction (all files) 2024-11-19T21:40:49,436 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/B in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,436 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/90a34601b6c7458db9fd62fde5aaab03, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/764cc11bfe884e47865f176eb8306b9c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3ad2c96e84e54e53a1716ccf7a8b4804] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=36.6 K 2024-11-19T21:40:49,436 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:49,437 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/A is initiating minor compaction (all files) 2024-11-19T21:40:49,437 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/A in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,437 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/ba8f857e786b4d67bbeb0c831b2ea1a0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7af7080d9db74c26912da486ead46670, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1e43d40525494198835abf0e98ec6f15] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=100.7 K 2024-11-19T21:40:49,437 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,437 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/ba8f857e786b4d67bbeb0c831b2ea1a0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7af7080d9db74c26912da486ead46670, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1e43d40525494198835abf0e98ec6f15] 2024-11-19T21:40:49,438 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba8f857e786b4d67bbeb0c831b2ea1a0, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732052445020 2024-11-19T21:40:49,439 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7af7080d9db74c26912da486ead46670, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732052446389 2024-11-19T21:40:49,439 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 90a34601b6c7458db9fd62fde5aaab03, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732052445020 2024-11-19T21:40:49,440 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e43d40525494198835abf0e98ec6f15, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732052447610 2024-11-19T21:40:49,441 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 764cc11bfe884e47865f176eb8306b9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732052446389 2024-11-19T21:40:49,444 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ad2c96e84e54e53a1716ccf7a8b4804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732052447618 2024-11-19T21:40:49,454 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:49,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:40:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:49,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119091585dc2c2a4508991eefc0ae92924c_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052449450/Put/seqid=0 2024-11-19T21:40:49,484 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#B#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:49,484 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/f81c9f0691644812b48db5c39aad4904 is 50, key is test_row_0/B:col10/1732052447618/Put/seqid=0 2024-11-19T21:40:49,486 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:49,504 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111907f96fe12e0f4a6394c10383018cfbfc_a4602ba7d48e7c270c08c7a6e512b385 store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:49,506 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111907f96fe12e0f4a6394c10383018cfbfc_a4602ba7d48e7c270c08c7a6e512b385, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:49,506 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111907f96fe12e0f4a6394c10383018cfbfc_a4602ba7d48e7c270c08c7a6e512b385 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:49,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742053_1229 (size=17534) 2024-11-19T21:40:49,511 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:49,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742054_1230 (size=13017) 2024-11-19T21:40:49,522 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119091585dc2c2a4508991eefc0ae92924c_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119091585dc2c2a4508991eefc0ae92924c_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:49,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/aa9e7c2c498f4af4a6ac7f15ffebf880, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:49,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/aa9e7c2c498f4af4a6ac7f15ffebf880 is 175, key is test_row_0/A:col10/1732052449450/Put/seqid=0 2024-11-19T21:40:49,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052509535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052509539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,543 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/f81c9f0691644812b48db5c39aad4904 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f81c9f0691644812b48db5c39aad4904 2024-11-19T21:40:49,563 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/B of a4602ba7d48e7c270c08c7a6e512b385 into f81c9f0691644812b48db5c39aad4904(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:49,564 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:49,564 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/B, priority=13, startTime=1732052449433; duration=0sec 2024-11-19T21:40:49,564 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:49,564 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:B 2024-11-19T21:40:49,564 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:49,566 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:49,566 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/C is initiating minor compaction (all files) 2024-11-19T21:40:49,567 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/C in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,567 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f4a7172f9574793883f35fe66e55a3a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/b13d30529cfa4b1ba4b2f8cace3ead8e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ef3dc3d01b0142ec9bedfba68795066d] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=36.6 K 2024-11-19T21:40:49,568 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f4a7172f9574793883f35fe66e55a3a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732052445020 2024-11-19T21:40:49,569 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b13d30529cfa4b1ba4b2f8cace3ead8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732052446389 2024-11-19T21:40:49,569 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting ef3dc3d01b0142ec9bedfba68795066d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732052447618 2024-11-19T21:40:49,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742056_1232 (size=48639) 2024-11-19T21:40:49,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/aa9e7c2c498f4af4a6ac7f15ffebf880 2024-11-19T21:40:49,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742055_1231 (size=4469) 2024-11-19T21:40:49,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/92f9ddc25c53415c967303437aaa2bcc is 50, key is test_row_0/B:col10/1732052449450/Put/seqid=0 2024-11-19T21:40:49,609 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#C#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:49,609 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/ac91c16fc1174d86940d300d30e3451d is 50, key is test_row_0/C:col10/1732052447618/Put/seqid=0 2024-11-19T21:40:49,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052509648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052509648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742058_1234 (size=13017) 2024-11-19T21:40:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742057_1233 (size=12301) 2024-11-19T21:40:49,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/92f9ddc25c53415c967303437aaa2bcc 2024-11-19T21:40:49,658 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/ac91c16fc1174d86940d300d30e3451d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ac91c16fc1174d86940d300d30e3451d 2024-11-19T21:40:49,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/90fcf452e5b44b5eaba99e4e6eb41a40 is 50, key is test_row_0/C:col10/1732052449450/Put/seqid=0 2024-11-19T21:40:49,664 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/C of a4602ba7d48e7c270c08c7a6e512b385 into ac91c16fc1174d86940d300d30e3451d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:49,664 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:49,664 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/C, priority=13, startTime=1732052449433; duration=0sec 2024-11-19T21:40:49,665 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:49,665 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:C 2024-11-19T21:40:49,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742059_1235 (size=12301) 2024-11-19T21:40:49,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:49,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:49,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052509858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052509858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:49,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:49,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:49,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T21:40:50,001 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#A#compaction#194 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:50,002 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/759cdbc977014df090d96b3f5a04fa15 is 175, key is test_row_0/A:col10/1732052447618/Put/seqid=0 2024-11-19T21:40:50,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742060_1236 (size=31971) 2024-11-19T21:40:50,029 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/759cdbc977014df090d96b3f5a04fa15 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/759cdbc977014df090d96b3f5a04fa15 2024-11-19T21:40:50,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:50,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:50,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,041 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/A of a4602ba7d48e7c270c08c7a6e512b385 into 759cdbc977014df090d96b3f5a04fa15(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:50,041 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:50,041 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/A, priority=13, startTime=1732052449433; duration=0sec 2024-11-19T21:40:50,041 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:50,041 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:A 2024-11-19T21:40:50,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/90fcf452e5b44b5eaba99e4e6eb41a40 2024-11-19T21:40:50,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/aa9e7c2c498f4af4a6ac7f15ffebf880 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aa9e7c2c498f4af4a6ac7f15ffebf880 2024-11-19T21:40:50,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aa9e7c2c498f4af4a6ac7f15ffebf880, entries=250, sequenceid=331, filesize=47.5 K 2024-11-19T21:40:50,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/92f9ddc25c53415c967303437aaa2bcc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/92f9ddc25c53415c967303437aaa2bcc 2024-11-19T21:40:50,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/92f9ddc25c53415c967303437aaa2bcc, entries=150, sequenceid=331, filesize=12.0 K 2024-11-19T21:40:50,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/90fcf452e5b44b5eaba99e4e6eb41a40 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/90fcf452e5b44b5eaba99e4e6eb41a40 2024-11-19T21:40:50,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/90fcf452e5b44b5eaba99e4e6eb41a40, entries=150, sequenceid=331, filesize=12.0 K 2024-11-19T21:40:50,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a4602ba7d48e7c270c08c7a6e512b385 in 684ms, sequenceid=331, compaction requested=false 2024-11-19T21:40:50,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:50,173 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:50,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:40:50,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196d9bc1b657504cde9fdcde61afbf212c_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052449536/Put/seqid=0 2024-11-19T21:40:50,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:50,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:50,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742061_1237 (size=17534) 2024-11-19T21:40:50,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052510198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:50,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052510200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:50,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052510308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052510308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:50,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:50,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:50,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:50,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:50,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052510519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052510519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:50,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:50,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196d9bc1b657504cde9fdcde61afbf212c_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196d9bc1b657504cde9fdcde61afbf212c_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:50,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/82e6492e57a04c8e97b6dfa629fb5437, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:50,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/82e6492e57a04c8e97b6dfa629fb5437 is 175, key is test_row_0/A:col10/1732052449536/Put/seqid=0 2024-11-19T21:40:50,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742062_1238 (size=48639) 2024-11-19T21:40:50,607 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/82e6492e57a04c8e97b6dfa629fb5437 2024-11-19T21:40:50,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/59c2730769fe4b7fb013014f16dfa476 is 50, key is test_row_0/B:col10/1732052449536/Put/seqid=0 2024-11-19T21:40:50,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742063_1239 (size=12301) 2024-11-19T21:40:50,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:50,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:50,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:50,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:50,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052510829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:50,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052510829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T21:40:50,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:50,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:50,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:50,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:40:51,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/59c2730769fe4b7fb013014f16dfa476 2024-11-19T21:40:51,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/457bf703b68a4246adeb1477f9123bf2 is 50, key is test_row_0/C:col10/1732052449536/Put/seqid=0 2024-11-19T21:40:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742064_1240 (size=12301) 2024-11-19T21:40:51,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/457bf703b68a4246adeb1477f9123bf2 2024-11-19T21:40:51,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/82e6492e57a04c8e97b6dfa629fb5437 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/82e6492e57a04c8e97b6dfa629fb5437 2024-11-19T21:40:51,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/82e6492e57a04c8e97b6dfa629fb5437, entries=250, sequenceid=359, filesize=47.5 K 2024-11-19T21:40:51,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/59c2730769fe4b7fb013014f16dfa476 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/59c2730769fe4b7fb013014f16dfa476 2024-11-19T21:40:51,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/59c2730769fe4b7fb013014f16dfa476, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T21:40:51,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/457bf703b68a4246adeb1477f9123bf2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/457bf703b68a4246adeb1477f9123bf2 2024-11-19T21:40:51,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/457bf703b68a4246adeb1477f9123bf2, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T21:40:51,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a4602ba7d48e7c270c08c7a6e512b385 in 908ms, sequenceid=359, compaction requested=true 2024-11-19T21:40:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:51,082 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:51,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:51,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:51,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:51,082 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:51,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:51,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:51,083 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:51,083 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:51,083 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/A is initiating minor compaction (all files) 2024-11-19T21:40:51,083 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/B is initiating minor compaction (all files) 2024-11-19T21:40:51,083 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/A in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:51,083 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/B in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:51,083 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/759cdbc977014df090d96b3f5a04fa15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aa9e7c2c498f4af4a6ac7f15ffebf880, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/82e6492e57a04c8e97b6dfa629fb5437] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=126.2 K 2024-11-19T21:40:51,083 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f81c9f0691644812b48db5c39aad4904, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/92f9ddc25c53415c967303437aaa2bcc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/59c2730769fe4b7fb013014f16dfa476] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=36.7 K 2024-11-19T21:40:51,083 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:51,083 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/759cdbc977014df090d96b3f5a04fa15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aa9e7c2c498f4af4a6ac7f15ffebf880, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/82e6492e57a04c8e97b6dfa629fb5437] 2024-11-19T21:40:51,083 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 759cdbc977014df090d96b3f5a04fa15, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732052447618 2024-11-19T21:40:51,083 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting f81c9f0691644812b48db5c39aad4904, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732052447618 2024-11-19T21:40:51,084 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9e7c2c498f4af4a6ac7f15ffebf880, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732052448799 2024-11-19T21:40:51,084 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f9ddc25c53415c967303437aaa2bcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732052448799 2024-11-19T21:40:51,084 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e6492e57a04c8e97b6dfa629fb5437, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732052449523 2024-11-19T21:40:51,084 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c2730769fe4b7fb013014f16dfa476, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732052449536 2024-11-19T21:40:51,095 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:51,098 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411197cb56cd219c140ed9b45c1f9e2a5f4c6_a4602ba7d48e7c270c08c7a6e512b385 store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:51,098 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#B#compaction#202 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:51,098 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/179a79071a9c4f5991936a4973b02ab1 is 50, key is test_row_0/B:col10/1732052449536/Put/seqid=0 2024-11-19T21:40:51,100 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411197cb56cd219c140ed9b45c1f9e2a5f4c6_a4602ba7d48e7c270c08c7a6e512b385, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:51,100 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197cb56cd219c140ed9b45c1f9e2a5f4c6_a4602ba7d48e7c270c08c7a6e512b385 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:51,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T21:40:51,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:51,113 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:40:51,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742065_1241 (size=13119) 2024-11-19T21:40:51,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742066_1242 (size=4469) 2024-11-19T21:40:51,119 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#A#compaction#201 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:51,119 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/8efd14fcda764137b04fbd461c13c0c8 is 175, key is test_row_0/A:col10/1732052449536/Put/seqid=0 2024-11-19T21:40:51,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dcc3ad98bc334647ad02f63260b9be6f_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052450196/Put/seqid=0 2024-11-19T21:40:51,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742067_1243 (size=32073) 2024-11-19T21:40:51,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742068_1244 (size=12454) 2024-11-19T21:40:51,138 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/8efd14fcda764137b04fbd461c13c0c8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/8efd14fcda764137b04fbd461c13c0c8 2024-11-19T21:40:51,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:51,143 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dcc3ad98bc334647ad02f63260b9be6f_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dcc3ad98bc334647ad02f63260b9be6f_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:51,143 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/A of a4602ba7d48e7c270c08c7a6e512b385 into 8efd14fcda764137b04fbd461c13c0c8(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:51,143 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:51,143 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/A, priority=13, startTime=1732052451081; duration=0sec 2024-11-19T21:40:51,144 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:51,144 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:A 2024-11-19T21:40:51,144 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:51,145 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:51,145 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/C is initiating minor compaction (all files) 2024-11-19T21:40:51,145 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/C in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/4e829ff7b44a481bb19008ddc4bed9cb, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:51,145 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ac91c16fc1174d86940d300d30e3451d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/90fcf452e5b44b5eaba99e4e6eb41a40, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/457bf703b68a4246adeb1477f9123bf2] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=36.7 K 2024-11-19T21:40:51,146 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac91c16fc1174d86940d300d30e3451d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732052447618 2024-11-19T21:40:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/4e829ff7b44a481bb19008ddc4bed9cb is 175, key is test_row_0/A:col10/1732052450196/Put/seqid=0 2024-11-19T21:40:51,146 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90fcf452e5b44b5eaba99e4e6eb41a40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732052448799 2024-11-19T21:40:51,146 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 457bf703b68a4246adeb1477f9123bf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732052449536 2024-11-19T21:40:51,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742069_1245 (size=31255) 2024-11-19T21:40:51,154 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#C#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:51,155 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/3010e6a90ce44085bb18cfeb42076b81 is 50, key is test_row_0/C:col10/1732052449536/Put/seqid=0 2024-11-19T21:40:51,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742070_1246 (size=13119) 2024-11-19T21:40:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:51,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. as already flushing 2024-11-19T21:40:51,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052511386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:51,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052511394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:51,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42136 deadline: 1732052511498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:51,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:40:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42140 deadline: 1732052511508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 2024-11-19T21:40:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:40:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:40:51,517 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/179a79071a9c4f5991936a4973b02ab1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/179a79071a9c4f5991936a4973b02ab1 2024-11-19T21:40:51,524 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/B of a4602ba7d48e7c270c08c7a6e512b385 into 179a79071a9c4f5991936a4973b02ab1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:51,524 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:51,524 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/B, priority=13, startTime=1732052451082; duration=0sec 2024-11-19T21:40:51,524 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:51,524 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:B 2024-11-19T21:40:51,551 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/4e829ff7b44a481bb19008ddc4bed9cb 2024-11-19T21:40:51,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/ab08a895e12c4a768eeeb89ad7fd9d44 is 50, key is test_row_0/B:col10/1732052450196/Put/seqid=0 2024-11-19T21:40:51,567 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/3010e6a90ce44085bb18cfeb42076b81 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3010e6a90ce44085bb18cfeb42076b81 2024-11-19T21:40:51,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742071_1247 (size=12301) 2024-11-19T21:40:51,576 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/ab08a895e12c4a768eeeb89ad7fd9d44 2024-11-19T21:40:51,583 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/C of a4602ba7d48e7c270c08c7a6e512b385 into 3010e6a90ce44085bb18cfeb42076b81(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:51,583 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:51,583 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/C, priority=13, startTime=1732052451082; duration=0sec 2024-11-19T21:40:51,583 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:51,583 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:C 2024-11-19T21:40:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/61dedc2214cd4971b93393f28e51f3bc is 50, key is test_row_0/C:col10/1732052450196/Put/seqid=0 2024-11-19T21:40:51,594 INFO [Thread-718 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-718. 2024-11-19T21:40:51,594 DEBUG [Thread-718 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:51,594 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:51,594 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,597 INFO [Thread-720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-720. 2024-11-19T21:40:51,597 DEBUG [Thread-720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:51,597 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,597 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:51,598 INFO [Thread-722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-722. 2024-11-19T21:40:51,598 DEBUG [Thread-722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:51,598 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,599 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:51,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742072_1248 (size=12301) 2024-11-19T21:40:51,600 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/61dedc2214cd4971b93393f28e51f3bc 2024-11-19T21:40:51,601 INFO [Thread-724 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-724. 2024-11-19T21:40:51,601 DEBUG [Thread-724 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:51,601 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,602 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:51,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/4e829ff7b44a481bb19008ddc4bed9cb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/4e829ff7b44a481bb19008ddc4bed9cb 2024-11-19T21:40:51,608 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/4e829ff7b44a481bb19008ddc4bed9cb, entries=150, sequenceid=370, filesize=30.5 K 2024-11-19T21:40:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/ab08a895e12c4a768eeeb89ad7fd9d44 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/ab08a895e12c4a768eeeb89ad7fd9d44 2024-11-19T21:40:51,613 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/ab08a895e12c4a768eeeb89ad7fd9d44, entries=150, sequenceid=370, filesize=12.0 K 2024-11-19T21:40:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/61dedc2214cd4971b93393f28e51f3bc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/61dedc2214cd4971b93393f28e51f3bc 2024-11-19T21:40:51,618 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/61dedc2214cd4971b93393f28e51f3bc, entries=150, sequenceid=370, filesize=12.0 K 2024-11-19T21:40:51,619 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a4602ba7d48e7c270c08c7a6e512b385 in 506ms, sequenceid=370, compaction requested=false 2024-11-19T21:40:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-19T21:40:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-19T21:40:51,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T21:40:51,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8280 sec 2024-11-19T21:40:51,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.8330 sec 2024-11-19T21:40:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:40:51,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T21:40:51,710 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-19T21:40:51,710 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:51,710 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,710 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:51,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fadb4cff2c3e4cb49d385771a32ff3c5_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052451708/Put/seqid=0 2024-11-19T21:40:51,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742073_1249 (size=12454) 2024-11-19T21:40:51,730 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-19T21:40:51,730 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:51,730 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,730 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:51,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:52,121 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:52,125 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fadb4cff2c3e4cb49d385771a32ff3c5_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fadb4cff2c3e4cb49d385771a32ff3c5_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:52,126 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fae151d18e3f46d9ba5c82b35afb86ec, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:52,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fae151d18e3f46d9ba5c82b35afb86ec is 175, key is test_row_0/A:col10/1732052451708/Put/seqid=0 2024-11-19T21:40:52,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742074_1250 (size=31255) 2024-11-19T21:40:52,276 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T21:40:52,532 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fae151d18e3f46d9ba5c82b35afb86ec 2024-11-19T21:40:52,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/e67bb1760c2b40b091e4db6f475330e7 is 50, key is test_row_0/B:col10/1732052451708/Put/seqid=0 2024-11-19T21:40:52,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742075_1251 (size=12301) 2024-11-19T21:40:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T21:40:52,949 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:52,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/e67bb1760c2b40b091e4db6f475330e7 2024-11-19T21:40:52,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/6ee63b85e5f8489e9119212e157f77e5 is 50, key is test_row_0/C:col10/1732052451708/Put/seqid=0 2024-11-19T21:40:52,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742076_1252 (size=12301) 2024-11-19T21:40:53,219 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-11-19T21:40:53,219 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:53,219 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:53,220 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:53,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:53,260 INFO [Thread-713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-713. 2024-11-19T21:40:53,260 DEBUG [Thread-713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:53,260 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:53,260 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:53,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:53,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/6ee63b85e5f8489e9119212e157f77e5 2024-11-19T21:40:53,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fae151d18e3f46d9ba5c82b35afb86ec as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fae151d18e3f46d9ba5c82b35afb86ec 2024-11-19T21:40:53,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fae151d18e3f46d9ba5c82b35afb86ec, entries=150, sequenceid=399, filesize=30.5 K 2024-11-19T21:40:53,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/e67bb1760c2b40b091e4db6f475330e7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/e67bb1760c2b40b091e4db6f475330e7 2024-11-19T21:40:53,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/e67bb1760c2b40b091e4db6f475330e7, entries=150, sequenceid=399, filesize=12.0 K 2024-11-19T21:40:53,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/6ee63b85e5f8489e9119212e157f77e5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6ee63b85e5f8489e9119212e157f77e5 2024-11-19T21:40:53,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6ee63b85e5f8489e9119212e157f77e5, entries=150, sequenceid=399, filesize=12.0 K 2024-11-19T21:40:53,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=20.13 KB/20610 for a4602ba7d48e7c270c08c7a6e512b385 in 1687ms, sequenceid=399, compaction requested=true 2024-11-19T21:40:53,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:53,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:40:53,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:53,398 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:53,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:40:53,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:53,398 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:53,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4602ba7d48e7c270c08c7a6e512b385:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:40:53,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:53,399 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:53,399 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:53,399 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/B is initiating minor compaction (all files) 2024-11-19T21:40:53,399 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/A is initiating minor compaction (all files) 2024-11-19T21:40:53,399 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/B in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:53,399 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/A in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:53,399 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/179a79071a9c4f5991936a4973b02ab1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/ab08a895e12c4a768eeeb89ad7fd9d44, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/e67bb1760c2b40b091e4db6f475330e7] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=36.8 K 2024-11-19T21:40:53,399 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/8efd14fcda764137b04fbd461c13c0c8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/4e829ff7b44a481bb19008ddc4bed9cb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fae151d18e3f46d9ba5c82b35afb86ec] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=92.4 K 2024-11-19T21:40:53,399 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:53,399 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/8efd14fcda764137b04fbd461c13c0c8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/4e829ff7b44a481bb19008ddc4bed9cb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fae151d18e3f46d9ba5c82b35afb86ec] 2024-11-19T21:40:53,399 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 179a79071a9c4f5991936a4973b02ab1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732052449536 2024-11-19T21:40:53,400 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8efd14fcda764137b04fbd461c13c0c8, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732052449536 2024-11-19T21:40:53,400 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting ab08a895e12c4a768eeeb89ad7fd9d44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732052450189 2024-11-19T21:40:53,400 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e829ff7b44a481bb19008ddc4bed9cb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732052450189 2024-11-19T21:40:53,400 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e67bb1760c2b40b091e4db6f475330e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732052451365 2024-11-19T21:40:53,400 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting fae151d18e3f46d9ba5c82b35afb86ec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732052451365 2024-11-19T21:40:53,408 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#B#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:53,408 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/2164fccb8afb43e9b76ff1f603559b09 is 50, key is test_row_0/B:col10/1732052451708/Put/seqid=0 2024-11-19T21:40:53,409 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:53,411 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191aad5653c37149649d01707efa47886e_a4602ba7d48e7c270c08c7a6e512b385 store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:53,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742077_1253 (size=13221) 2024-11-19T21:40:53,425 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191aad5653c37149649d01707efa47886e_a4602ba7d48e7c270c08c7a6e512b385, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:53,425 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191aad5653c37149649d01707efa47886e_a4602ba7d48e7c270c08c7a6e512b385 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:53,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742078_1254 (size=4469) 2024-11-19T21:40:53,818 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/2164fccb8afb43e9b76ff1f603559b09 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/2164fccb8afb43e9b76ff1f603559b09 2024-11-19T21:40:53,824 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/B of a4602ba7d48e7c270c08c7a6e512b385 into 2164fccb8afb43e9b76ff1f603559b09(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:53,824 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:53,824 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/B, priority=13, startTime=1732052453398; duration=0sec 2024-11-19T21:40:53,824 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:40:53,824 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:B 2024-11-19T21:40:53,824 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:40:53,826 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:40:53,826 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a4602ba7d48e7c270c08c7a6e512b385/C is initiating minor compaction (all files) 2024-11-19T21:40:53,826 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4602ba7d48e7c270c08c7a6e512b385/C in TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:53,826 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3010e6a90ce44085bb18cfeb42076b81, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/61dedc2214cd4971b93393f28e51f3bc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6ee63b85e5f8489e9119212e157f77e5] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp, totalSize=36.8 K 2024-11-19T21:40:53,826 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3010e6a90ce44085bb18cfeb42076b81, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732052449536 2024-11-19T21:40:53,827 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 61dedc2214cd4971b93393f28e51f3bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732052450189 2024-11-19T21:40:53,827 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee63b85e5f8489e9119212e157f77e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732052451365 2024-11-19T21:40:53,830 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#A#compaction#211 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:53,831 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/10ee14d50f764f54be117a7204fdda89 is 175, key is test_row_0/A:col10/1732052451708/Put/seqid=0 2024-11-19T21:40:53,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742079_1255 (size=32175) 2024-11-19T21:40:53,836 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4602ba7d48e7c270c08c7a6e512b385#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:40:53,837 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/5712c1f73d484cd38497b55dee39c3d5 is 50, key is test_row_0/C:col10/1732052451708/Put/seqid=0 2024-11-19T21:40:53,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742080_1256 (size=13221) 2024-11-19T21:40:54,243 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/10ee14d50f764f54be117a7204fdda89 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10ee14d50f764f54be117a7204fdda89 2024-11-19T21:40:54,251 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/5712c1f73d484cd38497b55dee39c3d5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/5712c1f73d484cd38497b55dee39c3d5 2024-11-19T21:40:54,251 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/A of a4602ba7d48e7c270c08c7a6e512b385 into 10ee14d50f764f54be117a7204fdda89(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:54,251 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:54,251 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/A, priority=13, startTime=1732052453398; duration=0sec 2024-11-19T21:40:54,251 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:54,251 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:A 2024-11-19T21:40:54,257 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4602ba7d48e7c270c08c7a6e512b385/C of a4602ba7d48e7c270c08c7a6e512b385 into 5712c1f73d484cd38497b55dee39c3d5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:40:54,257 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4602ba7d48e7c270c08c7a6e512b385: 2024-11-19T21:40:54,257 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., storeName=a4602ba7d48e7c270c08c7a6e512b385/C, priority=13, startTime=1732052453398; duration=0sec 2024-11-19T21:40:54,257 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:40:54,257 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4602ba7d48e7c270c08c7a6e512b385:C 2024-11-19T21:40:56,451 INFO [Thread-715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-715. 2024-11-19T21:40:56,451 DEBUG [Thread-715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:40:56,452 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:56,452 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:56,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:56,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T21:40:56,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-19T21:40:56,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-19T21:40:56,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-19T21:40:56,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-19T21:40:56,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-19T21:40:56,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T21:40:56,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3260 2024-11-19T21:40:56,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3291 2024-11-19T21:40:56,453 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T21:40:56,453 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1446 2024-11-19T21:40:56,453 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4338 rows 2024-11-19T21:40:56,453 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1452 2024-11-19T21:40:56,453 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4356 rows 2024-11-19T21:40:56,453 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:40:56,453 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:40:56,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:56,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:40:56,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:40:56,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T21:40:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T21:40:56,461 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052456460"}]},"ts":"1732052456460"} 2024-11-19T21:40:56,463 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T21:40:56,463 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T21:40:56,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T21:40:56,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, UNASSIGN}] 2024-11-19T21:40:56,465 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, UNASSIGN 2024-11-19T21:40:56,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a4602ba7d48e7c270c08c7a6e512b385, regionState=CLOSING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:56,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, UNASSIGN because future has completed 2024-11-19T21:40:56,467 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T21:40:56,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:40:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T21:40:56,620 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:56,620 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T21:40:56,621 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing a4602ba7d48e7c270c08c7a6e512b385, disabling compactions & flushes 2024-11-19T21:40:56,621 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:56,621 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:56,621 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. after waiting 0 ms 2024-11-19T21:40:56,621 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:56,621 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing a4602ba7d48e7c270c08c7a6e512b385 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-19T21:40:56,632 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ef74bed3fbb44483aa82d9b7a3c6875a_a4602ba7d48e7c270c08c7a6e512b385 is 50, key is test_row_0/A:col10/1732052453218/Put/seqid=0 2024-11-19T21:40:56,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742081_1257 (size=9914) 2024-11-19T21:40:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T21:40:57,037 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:57,042 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ef74bed3fbb44483aa82d9b7a3c6875a_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef74bed3fbb44483aa82d9b7a3c6875a_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:57,043 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fbe1454a48cd4d16aa7e2faedf930060, store: [table=TestAcidGuarantees family=A region=a4602ba7d48e7c270c08c7a6e512b385] 2024-11-19T21:40:57,044 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fbe1454a48cd4d16aa7e2faedf930060 is 175, key is test_row_0/A:col10/1732052453218/Put/seqid=0 2024-11-19T21:40:57,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742082_1258 (size=22561) 2024-11-19T21:40:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T21:40:57,451 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fbe1454a48cd4d16aa7e2faedf930060 2024-11-19T21:40:57,465 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/60f4ee855b5743198e7015bf0acc8dd9 is 50, key is test_row_0/B:col10/1732052453218/Put/seqid=0 2024-11-19T21:40:57,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742083_1259 (size=9857) 2024-11-19T21:40:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T21:40:57,871 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/60f4ee855b5743198e7015bf0acc8dd9 2024-11-19T21:40:57,884 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/f092efc0b5c5418cacb8c74101c0f3ba is 50, key is test_row_0/C:col10/1732052453218/Put/seqid=0 2024-11-19T21:40:57,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742084_1260 (size=9857) 2024-11-19T21:40:58,289 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/f092efc0b5c5418cacb8c74101c0f3ba 2024-11-19T21:40:58,293 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/A/fbe1454a48cd4d16aa7e2faedf930060 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe1454a48cd4d16aa7e2faedf930060 2024-11-19T21:40:58,297 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe1454a48cd4d16aa7e2faedf930060, entries=100, sequenceid=409, filesize=22.0 K 2024-11-19T21:40:58,298 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/B/60f4ee855b5743198e7015bf0acc8dd9 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/60f4ee855b5743198e7015bf0acc8dd9 2024-11-19T21:40:58,301 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/60f4ee855b5743198e7015bf0acc8dd9, entries=100, sequenceid=409, filesize=9.6 K 2024-11-19T21:40:58,302 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/.tmp/C/f092efc0b5c5418cacb8c74101c0f3ba as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f092efc0b5c5418cacb8c74101c0f3ba 2024-11-19T21:40:58,307 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f092efc0b5c5418cacb8c74101c0f3ba, entries=100, sequenceid=409, filesize=9.6 K 2024-11-19T21:40:58,308 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a4602ba7d48e7c270c08c7a6e512b385 in 1687ms, sequenceid=409, compaction requested=false 2024-11-19T21:40:58,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10b8995430154f54a69f89a9a7b58580, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/30747fb13f6a4bd18bd8bcb88b74cd88, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1b79767888ba422bb4f8a045bd0d7bbc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3916a52022cd4b35b7b9525ee3b57ad5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/5acea5a031844d478f9d655c42a93107, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/c9278ad2c5374e3981cb93cb4625e5f9, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/d01337bc72834d0aa108f3b88ab71dd4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/0a47d012a12c45629d53a16e430d1f67, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/43c7b6eb0cae470ca4dc2a93c1730446, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3be002c27e2a4369aa57a8837e910da6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7c55b753444249e7a5c9a5f28af43b5f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/21c99c5eaa884a61a6b7f8e8e21f7296, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1622b080d7284fedb1b1e07bbe6f5c2a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aeec80f101c94abf92c059dcd02b2d01, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe5129e4e314de09d46c3fe4fdc0c78, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/327b771195194c82b34d0cdb5489f941, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/598fcbfaadd34a06b6e7b19744e1e8b4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/af5f63dce63e4684961b911a561bdc15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/ba8f857e786b4d67bbeb0c831b2ea1a0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7af7080d9db74c26912da486ead46670, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1e43d40525494198835abf0e98ec6f15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/759cdbc977014df090d96b3f5a04fa15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aa9e7c2c498f4af4a6ac7f15ffebf880, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/82e6492e57a04c8e97b6dfa629fb5437, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/8efd14fcda764137b04fbd461c13c0c8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/4e829ff7b44a481bb19008ddc4bed9cb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fae151d18e3f46d9ba5c82b35afb86ec] to archive 2024-11-19T21:40:58,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:40:58,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10b8995430154f54a69f89a9a7b58580 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10b8995430154f54a69f89a9a7b58580 2024-11-19T21:40:58,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/30747fb13f6a4bd18bd8bcb88b74cd88 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/30747fb13f6a4bd18bd8bcb88b74cd88 2024-11-19T21:40:58,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1b79767888ba422bb4f8a045bd0d7bbc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1b79767888ba422bb4f8a045bd0d7bbc 2024-11-19T21:40:58,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3916a52022cd4b35b7b9525ee3b57ad5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3916a52022cd4b35b7b9525ee3b57ad5 2024-11-19T21:40:58,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/5acea5a031844d478f9d655c42a93107 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/5acea5a031844d478f9d655c42a93107 2024-11-19T21:40:58,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/c9278ad2c5374e3981cb93cb4625e5f9 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/c9278ad2c5374e3981cb93cb4625e5f9 2024-11-19T21:40:58,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/d01337bc72834d0aa108f3b88ab71dd4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/d01337bc72834d0aa108f3b88ab71dd4 2024-11-19T21:40:58,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/0a47d012a12c45629d53a16e430d1f67 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/0a47d012a12c45629d53a16e430d1f67 2024-11-19T21:40:58,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/43c7b6eb0cae470ca4dc2a93c1730446 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/43c7b6eb0cae470ca4dc2a93c1730446 2024-11-19T21:40:58,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3be002c27e2a4369aa57a8837e910da6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/3be002c27e2a4369aa57a8837e910da6 2024-11-19T21:40:58,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7c55b753444249e7a5c9a5f28af43b5f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7c55b753444249e7a5c9a5f28af43b5f 2024-11-19T21:40:58,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/21c99c5eaa884a61a6b7f8e8e21f7296 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/21c99c5eaa884a61a6b7f8e8e21f7296 2024-11-19T21:40:58,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1622b080d7284fedb1b1e07bbe6f5c2a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1622b080d7284fedb1b1e07bbe6f5c2a 2024-11-19T21:40:58,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aeec80f101c94abf92c059dcd02b2d01 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aeec80f101c94abf92c059dcd02b2d01 2024-11-19T21:40:58,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe5129e4e314de09d46c3fe4fdc0c78 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe5129e4e314de09d46c3fe4fdc0c78 2024-11-19T21:40:58,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/327b771195194c82b34d0cdb5489f941 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/327b771195194c82b34d0cdb5489f941 2024-11-19T21:40:58,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/598fcbfaadd34a06b6e7b19744e1e8b4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/598fcbfaadd34a06b6e7b19744e1e8b4 2024-11-19T21:40:58,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/af5f63dce63e4684961b911a561bdc15 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/af5f63dce63e4684961b911a561bdc15 2024-11-19T21:40:58,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/ba8f857e786b4d67bbeb0c831b2ea1a0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/ba8f857e786b4d67bbeb0c831b2ea1a0 2024-11-19T21:40:58,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7af7080d9db74c26912da486ead46670 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/7af7080d9db74c26912da486ead46670 2024-11-19T21:40:58,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1e43d40525494198835abf0e98ec6f15 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/1e43d40525494198835abf0e98ec6f15 2024-11-19T21:40:58,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/759cdbc977014df090d96b3f5a04fa15 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/759cdbc977014df090d96b3f5a04fa15 2024-11-19T21:40:58,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aa9e7c2c498f4af4a6ac7f15ffebf880 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/aa9e7c2c498f4af4a6ac7f15ffebf880 2024-11-19T21:40:58,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/82e6492e57a04c8e97b6dfa629fb5437 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/82e6492e57a04c8e97b6dfa629fb5437 2024-11-19T21:40:58,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/8efd14fcda764137b04fbd461c13c0c8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/8efd14fcda764137b04fbd461c13c0c8 2024-11-19T21:40:58,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/4e829ff7b44a481bb19008ddc4bed9cb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/4e829ff7b44a481bb19008ddc4bed9cb 2024-11-19T21:40:58,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fae151d18e3f46d9ba5c82b35afb86ec to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fae151d18e3f46d9ba5c82b35afb86ec 2024-11-19T21:40:58,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/673409d47aa241169538e6c763833a6c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/82ab22c475ba4a209bf0309fa153d136, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d2bbf932ea4848b19db096ed67c38cec, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/382c012f4e1b4440ae6c84508238bfbc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/97e26ca938ab45aab48bb01ba7be86e7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/7174eda8a6ea443fab79a4606bf56581, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3cf90be655ad481086fe481694d3013a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4a1728ead9b472981c74649d5afe5cc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/8c152bca2ea04efbb3a2ab0a89b5617c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f37bc3f8d227435d9051829f566b8b20, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/5669f7d5246c4af9ae92d58537f8cb35, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c36037f4800f496694d0e112b9fdfe92, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b81637017e1f4f208b79997d11bd9c8f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4f65a9825fc4afcbddd67d0a2a42c9c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c30936cbb59141bb831c1f376b2aa179, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/6e92d8a899ff417fbdeb1777030e2c13, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b925d723b79d4dcab41a47b8e7933889, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/90a34601b6c7458db9fd62fde5aaab03, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/510d894c530041c981411191562fb627, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/764cc11bfe884e47865f176eb8306b9c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f81c9f0691644812b48db5c39aad4904, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3ad2c96e84e54e53a1716ccf7a8b4804, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/92f9ddc25c53415c967303437aaa2bcc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/179a79071a9c4f5991936a4973b02ab1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/59c2730769fe4b7fb013014f16dfa476, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/ab08a895e12c4a768eeeb89ad7fd9d44, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/e67bb1760c2b40b091e4db6f475330e7] to archive 2024-11-19T21:40:58,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:40:58,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/673409d47aa241169538e6c763833a6c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/673409d47aa241169538e6c763833a6c 2024-11-19T21:40:58,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/82ab22c475ba4a209bf0309fa153d136 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/82ab22c475ba4a209bf0309fa153d136 2024-11-19T21:40:58,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d2bbf932ea4848b19db096ed67c38cec to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d2bbf932ea4848b19db096ed67c38cec 2024-11-19T21:40:58,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/382c012f4e1b4440ae6c84508238bfbc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/382c012f4e1b4440ae6c84508238bfbc 2024-11-19T21:40:58,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/97e26ca938ab45aab48bb01ba7be86e7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/97e26ca938ab45aab48bb01ba7be86e7 2024-11-19T21:40:58,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/7174eda8a6ea443fab79a4606bf56581 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/7174eda8a6ea443fab79a4606bf56581 2024-11-19T21:40:58,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3cf90be655ad481086fe481694d3013a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3cf90be655ad481086fe481694d3013a 2024-11-19T21:40:58,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4a1728ead9b472981c74649d5afe5cc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4a1728ead9b472981c74649d5afe5cc 2024-11-19T21:40:58,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/8c152bca2ea04efbb3a2ab0a89b5617c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/8c152bca2ea04efbb3a2ab0a89b5617c 2024-11-19T21:40:58,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f37bc3f8d227435d9051829f566b8b20 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f37bc3f8d227435d9051829f566b8b20 2024-11-19T21:40:58,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/5669f7d5246c4af9ae92d58537f8cb35 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/5669f7d5246c4af9ae92d58537f8cb35 2024-11-19T21:40:58,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c36037f4800f496694d0e112b9fdfe92 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c36037f4800f496694d0e112b9fdfe92 2024-11-19T21:40:58,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b81637017e1f4f208b79997d11bd9c8f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b81637017e1f4f208b79997d11bd9c8f 2024-11-19T21:40:58,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4f65a9825fc4afcbddd67d0a2a42c9c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/d4f65a9825fc4afcbddd67d0a2a42c9c 2024-11-19T21:40:58,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c30936cbb59141bb831c1f376b2aa179 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/c30936cbb59141bb831c1f376b2aa179 2024-11-19T21:40:58,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/6e92d8a899ff417fbdeb1777030e2c13 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/6e92d8a899ff417fbdeb1777030e2c13 2024-11-19T21:40:58,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b925d723b79d4dcab41a47b8e7933889 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/b925d723b79d4dcab41a47b8e7933889 2024-11-19T21:40:58,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/90a34601b6c7458db9fd62fde5aaab03 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/90a34601b6c7458db9fd62fde5aaab03 2024-11-19T21:40:58,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/510d894c530041c981411191562fb627 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/510d894c530041c981411191562fb627 2024-11-19T21:40:58,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/764cc11bfe884e47865f176eb8306b9c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/764cc11bfe884e47865f176eb8306b9c 2024-11-19T21:40:58,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f81c9f0691644812b48db5c39aad4904 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/f81c9f0691644812b48db5c39aad4904 2024-11-19T21:40:58,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3ad2c96e84e54e53a1716ccf7a8b4804 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/3ad2c96e84e54e53a1716ccf7a8b4804 2024-11-19T21:40:58,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/92f9ddc25c53415c967303437aaa2bcc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/92f9ddc25c53415c967303437aaa2bcc 2024-11-19T21:40:58,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/179a79071a9c4f5991936a4973b02ab1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/179a79071a9c4f5991936a4973b02ab1 2024-11-19T21:40:58,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/59c2730769fe4b7fb013014f16dfa476 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/59c2730769fe4b7fb013014f16dfa476 2024-11-19T21:40:58,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/ab08a895e12c4a768eeeb89ad7fd9d44 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/ab08a895e12c4a768eeeb89ad7fd9d44 2024-11-19T21:40:58,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/e67bb1760c2b40b091e4db6f475330e7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/e67bb1760c2b40b091e4db6f475330e7 2024-11-19T21:40:58,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35debe76cca443afbb5323e3854c2ea5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/478432070f614832b4b9772389227f63, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/157df3f2d8e4412dbea23e1dc775f1ea, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/cc7b4ce83c614a18a1f14b8002aa380d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f3229ff773984b87a2531f96f5e8e75e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/24171bfe5d5047bc87af07acf48e257e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f40d56430fa4bdd82ef788d56edbd69, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2ae0f806835b4eb89968a5dbfef630e8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/936066e7a7774e8dabdf0e7a07f66dbd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2f33c3789313445c9597eb46802660af, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/0eab3f873d2540339a0987dfabcb9c7a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/bc5f680225b34dda8f32e438ca1ec408, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/290e280faf6541d3836df791e0bdb450, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35d26be9fef2490bb3e706781888620f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/4437df20a0494d42b21095cb58533fbf, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3f2b906fdf234200b27841104023acc7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6c2efb9fa35e449cb917ff87f028449b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f4a7172f9574793883f35fe66e55a3a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/38aa6b2e99204e0bb643f4140ee44a8b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/b13d30529cfa4b1ba4b2f8cace3ead8e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ac91c16fc1174d86940d300d30e3451d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ef3dc3d01b0142ec9bedfba68795066d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/90fcf452e5b44b5eaba99e4e6eb41a40, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3010e6a90ce44085bb18cfeb42076b81, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/457bf703b68a4246adeb1477f9123bf2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/61dedc2214cd4971b93393f28e51f3bc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6ee63b85e5f8489e9119212e157f77e5] to archive 2024-11-19T21:40:58,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:40:58,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35debe76cca443afbb5323e3854c2ea5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35debe76cca443afbb5323e3854c2ea5 2024-11-19T21:40:58,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/478432070f614832b4b9772389227f63 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/478432070f614832b4b9772389227f63 2024-11-19T21:40:58,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/157df3f2d8e4412dbea23e1dc775f1ea to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/157df3f2d8e4412dbea23e1dc775f1ea 2024-11-19T21:40:58,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/cc7b4ce83c614a18a1f14b8002aa380d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/cc7b4ce83c614a18a1f14b8002aa380d 2024-11-19T21:40:58,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f3229ff773984b87a2531f96f5e8e75e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f3229ff773984b87a2531f96f5e8e75e 2024-11-19T21:40:58,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/24171bfe5d5047bc87af07acf48e257e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/24171bfe5d5047bc87af07acf48e257e 2024-11-19T21:40:58,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f40d56430fa4bdd82ef788d56edbd69 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f40d56430fa4bdd82ef788d56edbd69 2024-11-19T21:40:58,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2ae0f806835b4eb89968a5dbfef630e8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2ae0f806835b4eb89968a5dbfef630e8 2024-11-19T21:40:58,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/936066e7a7774e8dabdf0e7a07f66dbd to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/936066e7a7774e8dabdf0e7a07f66dbd 2024-11-19T21:40:58,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2f33c3789313445c9597eb46802660af to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/2f33c3789313445c9597eb46802660af 2024-11-19T21:40:58,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/0eab3f873d2540339a0987dfabcb9c7a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/0eab3f873d2540339a0987dfabcb9c7a 2024-11-19T21:40:58,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/bc5f680225b34dda8f32e438ca1ec408 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/bc5f680225b34dda8f32e438ca1ec408 2024-11-19T21:40:58,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/290e280faf6541d3836df791e0bdb450 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/290e280faf6541d3836df791e0bdb450 2024-11-19T21:40:58,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35d26be9fef2490bb3e706781888620f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/35d26be9fef2490bb3e706781888620f 2024-11-19T21:40:58,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/4437df20a0494d42b21095cb58533fbf to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/4437df20a0494d42b21095cb58533fbf 2024-11-19T21:40:58,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3f2b906fdf234200b27841104023acc7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3f2b906fdf234200b27841104023acc7 2024-11-19T21:40:58,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6c2efb9fa35e449cb917ff87f028449b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6c2efb9fa35e449cb917ff87f028449b 2024-11-19T21:40:58,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f4a7172f9574793883f35fe66e55a3a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/9f4a7172f9574793883f35fe66e55a3a 2024-11-19T21:40:58,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/38aa6b2e99204e0bb643f4140ee44a8b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/38aa6b2e99204e0bb643f4140ee44a8b 2024-11-19T21:40:58,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/b13d30529cfa4b1ba4b2f8cace3ead8e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/b13d30529cfa4b1ba4b2f8cace3ead8e 2024-11-19T21:40:58,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ac91c16fc1174d86940d300d30e3451d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ac91c16fc1174d86940d300d30e3451d 2024-11-19T21:40:58,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ef3dc3d01b0142ec9bedfba68795066d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/ef3dc3d01b0142ec9bedfba68795066d 2024-11-19T21:40:58,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/90fcf452e5b44b5eaba99e4e6eb41a40 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/90fcf452e5b44b5eaba99e4e6eb41a40 2024-11-19T21:40:58,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3010e6a90ce44085bb18cfeb42076b81 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/3010e6a90ce44085bb18cfeb42076b81 2024-11-19T21:40:58,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/457bf703b68a4246adeb1477f9123bf2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/457bf703b68a4246adeb1477f9123bf2 2024-11-19T21:40:58,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/61dedc2214cd4971b93393f28e51f3bc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/61dedc2214cd4971b93393f28e51f3bc 2024-11-19T21:40:58,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6ee63b85e5f8489e9119212e157f77e5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/6ee63b85e5f8489e9119212e157f77e5 2024-11-19T21:40:58,394 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/recovered.edits/412.seqid, newMaxSeqId=412, maxSeqId=4 2024-11-19T21:40:58,395 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. 2024-11-19T21:40:58,395 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for a4602ba7d48e7c270c08c7a6e512b385: Waiting for close lock at 1732052456621Running coprocessor pre-close hooks at 1732052456621Disabling compacts and flushes for region at 1732052456621Disabling writes for close at 1732052456621Obtaining lock to block concurrent updates at 1732052456621Preparing flush snapshotting stores in a4602ba7d48e7c270c08c7a6e512b385 at 1732052456621Finished memstore snapshotting TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732052456622 (+1 ms)Flushing stores of TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385. at 1732052456624 (+2 ms)Flushing a4602ba7d48e7c270c08c7a6e512b385/A: creating writer at 1732052456624Flushing mob file a4602ba7d48e7c270c08c7a6e512b385/A: appending metadata at 1732052456632 (+8 ms)Flushing mob file a4602ba7d48e7c270c08c7a6e512b385/A: closing flushed file at 1732052456632Flushing a4602ba7d48e7c270c08c7a6e512b385/A: appending metadata at 1732052457043 (+411 ms)Flushing a4602ba7d48e7c270c08c7a6e512b385/A: closing flushed file at 1732052457044 (+1 ms)Flushing a4602ba7d48e7c270c08c7a6e512b385/B: creating writer at 1732052457460 (+416 ms)Flushing a4602ba7d48e7c270c08c7a6e512b385/B: appending metadata at 1732052457464 (+4 ms)Flushing a4602ba7d48e7c270c08c7a6e512b385/B: closing flushed file at 1732052457464Flushing a4602ba7d48e7c270c08c7a6e512b385/C: creating writer at 1732052457880 (+416 ms)Flushing a4602ba7d48e7c270c08c7a6e512b385/C: appending metadata at 1732052457883 (+3 ms)Flushing a4602ba7d48e7c270c08c7a6e512b385/C: closing flushed file at 1732052457883Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11a3cb85: reopening flushed file at 1732052458293 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@468524e8: reopening flushed file at 1732052458297 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d9499ea: reopening flushed file at 1732052458301 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a4602ba7d48e7c270c08c7a6e512b385 in 1687ms, sequenceid=409, compaction requested=false at 1732052458308 (+7 ms)Writing region close event to WAL at 1732052458391 (+83 ms)Running coprocessor post-close hooks at 1732052458395 (+4 ms)Closed at 1732052458395 2024-11-19T21:40:58,397 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,397 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a4602ba7d48e7c270c08c7a6e512b385, regionState=CLOSED 2024-11-19T21:40:58,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:40:58,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T21:40:58,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure a4602ba7d48e7c270c08c7a6e512b385, server=707262ffa86d,42821,1732052394005 in 1.9320 sec 2024-11-19T21:40:58,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-19T21:40:58,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4602ba7d48e7c270c08c7a6e512b385, UNASSIGN in 1.9370 sec 2024-11-19T21:40:58,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T21:40:58,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9400 sec 2024-11-19T21:40:58,406 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052458406"}]},"ts":"1732052458406"} 2024-11-19T21:40:58,407 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T21:40:58,407 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T21:40:58,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9530 sec 2024-11-19T21:40:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T21:40:58,608 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:58,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T21:40:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:58,610 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T21:40:58,611 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:58,613 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,616 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/recovered.edits] 2024-11-19T21:40:58,618 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10ee14d50f764f54be117a7204fdda89 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/10ee14d50f764f54be117a7204fdda89 2024-11-19T21:40:58,620 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe1454a48cd4d16aa7e2faedf930060 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/A/fbe1454a48cd4d16aa7e2faedf930060 2024-11-19T21:40:58,622 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/2164fccb8afb43e9b76ff1f603559b09 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/2164fccb8afb43e9b76ff1f603559b09 2024-11-19T21:40:58,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/60f4ee855b5743198e7015bf0acc8dd9 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/B/60f4ee855b5743198e7015bf0acc8dd9 2024-11-19T21:40:58,625 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/5712c1f73d484cd38497b55dee39c3d5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/5712c1f73d484cd38497b55dee39c3d5 2024-11-19T21:40:58,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f092efc0b5c5418cacb8c74101c0f3ba to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/C/f092efc0b5c5418cacb8c74101c0f3ba 2024-11-19T21:40:58,628 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/recovered.edits/412.seqid to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385/recovered.edits/412.seqid 2024-11-19T21:40:58,628 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,628 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T21:40:58,629 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T21:40:58,629 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T21:40:58,632 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119091585dc2c2a4508991eefc0ae92924c_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119091585dc2c2a4508991eefc0ae92924c_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,633 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190ab120e5c48a42c3b77b27dec1da6772_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190ab120e5c48a42c3b77b27dec1da6772_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,634 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d74ca6fbd3947c2a306f308e3ed0e46_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190d74ca6fbd3947c2a306f308e3ed0e46_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,635 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111911501b48d3d94924ad18559f1777dcb5_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111911501b48d3d94924ad18559f1777dcb5_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111912c062dabd764797b06124dbc8b5ac7f_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111912c062dabd764797b06124dbc8b5ac7f_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,637 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111922a821cdf6d147229713cae217314017_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111922a821cdf6d147229713cae217314017_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192c0dee9a34ca4836970a430698760356_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192c0dee9a34ca4836970a430698760356_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,639 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111948cc22ee59e7411981f5a02f83101c32_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111948cc22ee59e7411981f5a02f83101c32_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d68eb991482411c948c5e417477ae0d_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d68eb991482411c948c5e417477ae0d_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,641 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196d9bc1b657504cde9fdcde61afbf212c_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196d9bc1b657504cde9fdcde61afbf212c_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196dd421a113f94be18807649398ca8f7e_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196dd421a113f94be18807649398ca8f7e_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a51046418547403e8e99a3c3b3a809b2_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a51046418547403e8e99a3c3b3a809b2_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d1d5bd285fe044d7b6992b3eb329d3a0_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d1d5bd285fe044d7b6992b3eb329d3a0_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d200a981de584ae281d125592a3f42fc_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d200a981de584ae281d125592a3f42fc_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d2b8a1705bd84766b695c729ff7fe04f_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d2b8a1705bd84766b695c729ff7fe04f_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dcc3ad98bc334647ad02f63260b9be6f_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dcc3ad98bc334647ad02f63260b9be6f_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e2302ed910cc47efbaf1bc529e5da374_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e2302ed910cc47efbaf1bc529e5da374_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,648 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef74bed3fbb44483aa82d9b7a3c6875a_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef74bed3fbb44483aa82d9b7a3c6875a_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f6f9a7ec75824b10af61aa8ef33dd211_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f6f9a7ec75824b10af61aa8ef33dd211_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fa9628492fb240d5ac7673fb6e313654_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fa9628492fb240d5ac7673fb6e313654_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,650 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fadb4cff2c3e4cb49d385771a32ff3c5_a4602ba7d48e7c270c08c7a6e512b385 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fadb4cff2c3e4cb49d385771a32ff3c5_a4602ba7d48e7c270c08c7a6e512b385 2024-11-19T21:40:58,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T21:40:58,652 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:58,654 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T21:40:58,656 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T21:40:58,657 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:58,657 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T21:40:58,657 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732052458657"}]},"ts":"9223372036854775807"} 2024-11-19T21:40:58,659 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T21:40:58,659 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a4602ba7d48e7c270c08c7a6e512b385, NAME => 'TestAcidGuarantees,,1732052428336.a4602ba7d48e7c270c08c7a6e512b385.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T21:40:58,659 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T21:40:58,660 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732052458659"}]},"ts":"9223372036854775807"} 2024-11-19T21:40:58,661 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T21:40:58,661 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:58,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-19T21:40:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T21:40:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T21:40:58,719 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:40:58,732 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239 (was 239), OpenFileDescriptor=457 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=507 (was 463) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4670 (was 5451) 2024-11-19T21:40:58,744 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=507, ProcessCount=11, AvailableMemoryMB=4670 2024-11-19T21:40:58,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T21:40:58,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:40:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T21:40:58,747 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T21:40:58,747 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:58,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-19T21:40:58,748 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T21:40:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-19T21:40:58,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742085_1261 (size=959) 2024-11-19T21:40:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-19T21:40:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-19T21:40:59,159 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b 2024-11-19T21:40:59,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742086_1262 (size=53) 2024-11-19T21:40:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-19T21:40:59,568 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:40:59,568 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d5c69b89f3a83a34d8bee2d0fed83b40, disabling compactions & flushes 2024-11-19T21:40:59,568 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:40:59,568 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:40:59,568 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. after waiting 0 ms 2024-11-19T21:40:59,568 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:40:59,569 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:40:59,569 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d5c69b89f3a83a34d8bee2d0fed83b40: Waiting for close lock at 1732052459568Disabling compacts and flushes for region at 1732052459568Disabling writes for close at 1732052459568Writing region close event to WAL at 1732052459568Closed at 1732052459568 2024-11-19T21:40:59,570 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T21:40:59,570 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732052459570"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732052459570"}]},"ts":"1732052459570"} 2024-11-19T21:40:59,572 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T21:40:59,573 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T21:40:59,573 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052459573"}]},"ts":"1732052459573"} 2024-11-19T21:40:59,575 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T21:40:59,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5c69b89f3a83a34d8bee2d0fed83b40, ASSIGN}] 2024-11-19T21:40:59,577 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5c69b89f3a83a34d8bee2d0fed83b40, ASSIGN 2024-11-19T21:40:59,577 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5c69b89f3a83a34d8bee2d0fed83b40, ASSIGN; state=OFFLINE, location=707262ffa86d,42821,1732052394005; forceNewPlan=false, retain=false 2024-11-19T21:40:59,729 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=d5c69b89f3a83a34d8bee2d0fed83b40, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:59,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5c69b89f3a83a34d8bee2d0fed83b40, ASSIGN because future has completed 2024-11-19T21:40:59,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:40:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-19T21:40:59,893 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:40:59,893 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:40:59,894 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,894 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:40:59,894 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,894 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,896 INFO [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,898 INFO [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5c69b89f3a83a34d8bee2d0fed83b40 columnFamilyName A 2024-11-19T21:40:59,898 DEBUG [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:59,899 INFO [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] regionserver.HStore(327): Store=d5c69b89f3a83a34d8bee2d0fed83b40/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:59,899 INFO [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,901 INFO [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5c69b89f3a83a34d8bee2d0fed83b40 columnFamilyName B 2024-11-19T21:40:59,901 DEBUG [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:59,901 INFO [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] regionserver.HStore(327): Store=d5c69b89f3a83a34d8bee2d0fed83b40/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:59,901 INFO [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,903 INFO [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5c69b89f3a83a34d8bee2d0fed83b40 columnFamilyName C 2024-11-19T21:40:59,903 DEBUG [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:40:59,904 INFO [StoreOpener-d5c69b89f3a83a34d8bee2d0fed83b40-1 {}] regionserver.HStore(327): Store=d5c69b89f3a83a34d8bee2d0fed83b40/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:40:59,904 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,905 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,905 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,907 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,907 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,908 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:40:59,910 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,914 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:40:59,914 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened d5c69b89f3a83a34d8bee2d0fed83b40; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69673054, jitterRate=0.03820940852165222}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:40:59,914 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:40:59,916 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for d5c69b89f3a83a34d8bee2d0fed83b40: Running coprocessor pre-open hook at 1732052459895Writing region info on filesystem at 1732052459895Initializing all the Stores at 1732052459896 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052459896Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052459896Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052459896Cleaning up temporary data from old regions at 1732052459907 (+11 ms)Running coprocessor post-open hooks at 1732052459914 (+7 ms)Region opened successfully at 1732052459916 (+2 ms) 2024-11-19T21:40:59,916 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., pid=65, masterSystemTime=1732052459886 2024-11-19T21:40:59,918 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:40:59,918 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:40:59,919 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=d5c69b89f3a83a34d8bee2d0fed83b40, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:40:59,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:40:59,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T21:40:59,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 in 189 msec 2024-11-19T21:40:59,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-19T21:40:59,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5c69b89f3a83a34d8bee2d0fed83b40, ASSIGN in 348 msec 2024-11-19T21:40:59,926 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T21:40:59,926 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052459926"}]},"ts":"1732052459926"} 2024-11-19T21:40:59,928 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T21:40:59,929 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T21:40:59,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-19T21:41:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-19T21:41:00,900 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:00,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c45ccf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab88169, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,909 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2e94b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,911 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,914 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T21:41:00,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 707262ffa86d,37541,1732052393169 2024-11-19T21:41:00,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c737a8d 2024-11-19T21:41:00,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T21:41:00,916 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T21:41:00,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20832dd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,918 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287191a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,919 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67e2efe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb66027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,922 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c067da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,923 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b0daa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b4a085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,925 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37680c07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,926 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45672f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f8f105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,928 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41038c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,930 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35da1677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaa706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,931 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28befad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,933 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da73720, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417b74fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,935 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e2ba3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e88cfb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0a17bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,938 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5eed25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5680407e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb595da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,941 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4697a994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,943 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e961e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@262694dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,945 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d5ff48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,946 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef2ac7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df95348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:00,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:00,948 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fdbbd86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:00,949 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa71565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:00,950 DEBUG [Thread-1219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,950 DEBUG [Thread-1214 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,950 DEBUG [Thread-1212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,950 DEBUG [Thread-1217 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:00,950 DEBUG [Thread-1210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,950 DEBUG [Thread-1223 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,951 DEBUG [Thread-1221 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,951 DEBUG [Thread-1208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,951 DEBUG [Thread-1206 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,951 DEBUG [Thread-1225 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-19T21:41:00,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T21:41:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:00,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:00,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:41:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:00,971 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:00,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:41:01,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/fe7b28f39986468c926cb8123f4d8487 is 50, key is test_row_0/A:col10/1732052460967/Put/seqid=0 2024-11-19T21:41:01,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052521001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052521003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052521006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052521011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052521011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742087_1263 (size=12001) 2024-11-19T21:41:01,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/fe7b28f39986468c926cb8123f4d8487 2024-11-19T21:41:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T21:41:01,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/5c4af768a7f2424499a17c296f3cf219 is 50, key is test_row_0/B:col10/1732052460967/Put/seqid=0 2024-11-19T21:41:01,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742088_1264 (size=12001) 2024-11-19T21:41:01,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/5c4af768a7f2424499a17c296f3cf219 2024-11-19T21:41:01,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T21:41:01,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:01,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052521118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052521118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052521118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052521129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052521129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/7e6c13d419e647dcafa4d0d1f3f57cf4 is 50, key is test_row_0/C:col10/1732052460967/Put/seqid=0 2024-11-19T21:41:01,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742089_1265 (size=12001) 2024-11-19T21:41:01,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T21:41:01,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:01,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T21:41:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052521328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052521328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052521328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052521338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052521338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T21:41:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T21:41:01,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:01,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T21:41:01,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/7e6c13d419e647dcafa4d0d1f3f57cf4 2024-11-19T21:41:01,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/fe7b28f39986468c926cb8123f4d8487 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/fe7b28f39986468c926cb8123f4d8487 2024-11-19T21:41:01,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/fe7b28f39986468c926cb8123f4d8487, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T21:41:01,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/5c4af768a7f2424499a17c296f3cf219 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/5c4af768a7f2424499a17c296f3cf219 2024-11-19T21:41:01,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/5c4af768a7f2424499a17c296f3cf219, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T21:41:01,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/7e6c13d419e647dcafa4d0d1f3f57cf4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e6c13d419e647dcafa4d0d1f3f57cf4 2024-11-19T21:41:01,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e6c13d419e647dcafa4d0d1f3f57cf4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T21:41:01,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d5c69b89f3a83a34d8bee2d0fed83b40 in 650ms, sequenceid=15, compaction requested=false 2024-11-19T21:41:01,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:01,641 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:01,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:41:01,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/999539e530184d3584b8e99661e8cb97 is 50, key is test_row_0/A:col10/1732052461641/Put/seqid=0 2024-11-19T21:41:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052521661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052521662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052521666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052521668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052521668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742090_1266 (size=14341) 2024-11-19T21:41:01,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/999539e530184d3584b8e99661e8cb97 2024-11-19T21:41:01,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7e078fbe33064c989590fb5d2d37709c is 50, key is test_row_0/B:col10/1732052461641/Put/seqid=0 2024-11-19T21:41:01,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742091_1267 (size=12001) 2024-11-19T21:41:01,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7e078fbe33064c989590fb5d2d37709c 2024-11-19T21:41:01,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/6d666cad349a451c832bef70157da3f3 is 50, key is test_row_0/C:col10/1732052461641/Put/seqid=0 2024-11-19T21:41:01,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T21:41:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742092_1268 (size=12001) 2024-11-19T21:41:01,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052521778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052521778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052521779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052521779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052521779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T21:41:01,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:01,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:01,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052521988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052521988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052521988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052521988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052521989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T21:41:02,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:02,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:02,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:02,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T21:41:02,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/6d666cad349a451c832bef70157da3f3 2024-11-19T21:41:02,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/999539e530184d3584b8e99661e8cb97 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/999539e530184d3584b8e99661e8cb97 2024-11-19T21:41:02,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/999539e530184d3584b8e99661e8cb97, entries=200, sequenceid=39, filesize=14.0 K 2024-11-19T21:41:02,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7e078fbe33064c989590fb5d2d37709c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7e078fbe33064c989590fb5d2d37709c 2024-11-19T21:41:02,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7e078fbe33064c989590fb5d2d37709c, entries=150, sequenceid=39, filesize=11.7 K 2024-11-19T21:41:02,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/6d666cad349a451c832bef70157da3f3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d666cad349a451c832bef70157da3f3 2024-11-19T21:41:02,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d666cad349a451c832bef70157da3f3, entries=150, sequenceid=39, filesize=11.7 K 2024-11-19T21:41:02,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for d5c69b89f3a83a34d8bee2d0fed83b40 in 532ms, sequenceid=39, compaction requested=false 2024-11-19T21:41:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:02,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T21:41:02,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:02,186 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:41:02,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/eee57d0ca2424d94915c1d39293a23e1 is 50, key is test_row_0/A:col10/1732052461666/Put/seqid=0 2024-11-19T21:41:02,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742093_1269 (size=12001) 2024-11-19T21:41:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:02,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052522337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052522337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052522339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052522340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052522341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052522448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052522449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052522449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052522449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052522449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,624 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/eee57d0ca2424d94915c1d39293a23e1 2024-11-19T21:41:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/4737dfcf2dae433ab7cb34f85e00341e is 50, key is test_row_0/B:col10/1732052461666/Put/seqid=0 2024-11-19T21:41:02,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742094_1270 (size=12001) 2024-11-19T21:41:02,648 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/4737dfcf2dae433ab7cb34f85e00341e 2024-11-19T21:41:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/2ca725f8269c441bb985336decc9d57e is 50, key is test_row_0/C:col10/1732052461666/Put/seqid=0 2024-11-19T21:41:02,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052522658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052522659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052522659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052522659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052522659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742095_1271 (size=12001) 2024-11-19T21:41:02,686 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/2ca725f8269c441bb985336decc9d57e 2024-11-19T21:41:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/eee57d0ca2424d94915c1d39293a23e1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/eee57d0ca2424d94915c1d39293a23e1 2024-11-19T21:41:02,696 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/eee57d0ca2424d94915c1d39293a23e1, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T21:41:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/4737dfcf2dae433ab7cb34f85e00341e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4737dfcf2dae433ab7cb34f85e00341e 2024-11-19T21:41:02,701 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4737dfcf2dae433ab7cb34f85e00341e, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T21:41:02,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/2ca725f8269c441bb985336decc9d57e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2ca725f8269c441bb985336decc9d57e 2024-11-19T21:41:02,709 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2ca725f8269c441bb985336decc9d57e, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T21:41:02,710 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for d5c69b89f3a83a34d8bee2d0fed83b40 in 523ms, sequenceid=52, compaction requested=true 2024-11-19T21:41:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-19T21:41:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-19T21:41:02,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T21:41:02,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7520 sec 2024-11-19T21:41:02,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7640 sec 2024-11-19T21:41:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:02,971 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:02,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:41:02,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/08322169dac3431fb579d0f4b2180c7c is 50, key is test_row_0/A:col10/1732052462335/Put/seqid=0 2024-11-19T21:41:02,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052522975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052522979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052522980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052522980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052522981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:02,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742096_1272 (size=12001) 2024-11-19T21:41:02,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/08322169dac3431fb579d0f4b2180c7c 2024-11-19T21:41:03,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/eec26297e9714ada93b53b5ae08336e6 is 50, key is test_row_0/B:col10/1732052462335/Put/seqid=0 2024-11-19T21:41:03,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742097_1273 (size=12001) 2024-11-19T21:41:03,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/eec26297e9714ada93b53b5ae08336e6 2024-11-19T21:41:03,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/2e680e7a08ec4eb1adfdd8f1ea37a423 is 50, key is test_row_0/C:col10/1732052462335/Put/seqid=0 2024-11-19T21:41:03,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742098_1274 (size=12001) 2024-11-19T21:41:03,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/2e680e7a08ec4eb1adfdd8f1ea37a423 2024-11-19T21:41:03,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/08322169dac3431fb579d0f4b2180c7c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/08322169dac3431fb579d0f4b2180c7c 2024-11-19T21:41:03,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/08322169dac3431fb579d0f4b2180c7c, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T21:41:03,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/eec26297e9714ada93b53b5ae08336e6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/eec26297e9714ada93b53b5ae08336e6 2024-11-19T21:41:03,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/eec26297e9714ada93b53b5ae08336e6, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T21:41:03,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/2e680e7a08ec4eb1adfdd8f1ea37a423 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2e680e7a08ec4eb1adfdd8f1ea37a423 2024-11-19T21:41:03,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2e680e7a08ec4eb1adfdd8f1ea37a423, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T21:41:03,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for d5c69b89f3a83a34d8bee2d0fed83b40 in 121ms, sequenceid=77, compaction requested=true 2024-11-19T21:41:03,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:03,093 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:41:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:03,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:03,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:03,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:03,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:03,093 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:41:03,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:03,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:03,093 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:03,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:41:03,096 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:41:03,096 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:03,096 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,096 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/5c4af768a7f2424499a17c296f3cf219, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7e078fbe33064c989590fb5d2d37709c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4737dfcf2dae433ab7cb34f85e00341e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/eec26297e9714ada93b53b5ae08336e6] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=46.9 K 2024-11-19T21:41:03,097 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:41:03,098 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:03,098 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,098 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/fe7b28f39986468c926cb8123f4d8487, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/999539e530184d3584b8e99661e8cb97, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/eee57d0ca2424d94915c1d39293a23e1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/08322169dac3431fb579d0f4b2180c7c] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=49.2 K 2024-11-19T21:41:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T21:41:03,100 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:03,100 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4af768a7f2424499a17c296f3cf219, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732052460965 2024-11-19T21:41:03,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T21:41:03,102 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T21:41:03,104 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e078fbe33064c989590fb5d2d37709c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732052461007 2024-11-19T21:41:03,104 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:03,104 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting fe7b28f39986468c926cb8123f4d8487, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732052460965 2024-11-19T21:41:03,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:03,105 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4737dfcf2dae433ab7cb34f85e00341e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732052461656 2024-11-19T21:41:03,106 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting eec26297e9714ada93b53b5ae08336e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732052462334 2024-11-19T21:41:03,106 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 999539e530184d3584b8e99661e8cb97, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732052460990 2024-11-19T21:41:03,108 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting eee57d0ca2424d94915c1d39293a23e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732052461656 2024-11-19T21:41:03,109 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 08322169dac3431fb579d0f4b2180c7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732052462334 2024-11-19T21:41:03,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/79f2294723b3443a85a8b74a385d730f is 50, key is test_row_0/A:col10/1732052462980/Put/seqid=0 2024-11-19T21:41:03,134 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:03,134 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7d279bb86c5d499c9932f805a796e1fd is 50, key is test_row_0/B:col10/1732052462335/Put/seqid=0 2024-11-19T21:41:03,149 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:03,149 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/b4ea200ddffc469abf4a1b7d371f6845 is 50, key is test_row_0/A:col10/1732052462335/Put/seqid=0 2024-11-19T21:41:03,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742099_1275 (size=14341) 2024-11-19T21:41:03,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742100_1276 (size=12139) 2024-11-19T21:41:03,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742101_1277 (size=12139) 2024-11-19T21:41:03,179 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/b4ea200ddffc469abf4a1b7d371f6845 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/b4ea200ddffc469abf4a1b7d371f6845 2024-11-19T21:41:03,188 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into b4ea200ddffc469abf4a1b7d371f6845(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:03,188 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:03,188 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=12, startTime=1732052463092; duration=0sec 2024-11-19T21:41:03,188 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:03,188 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:03,189 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:41:03,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052523140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052523190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052523190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052523190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,196 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:41:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052523192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,196 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,197 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,197 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e6c13d419e647dcafa4d0d1f3f57cf4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d666cad349a451c832bef70157da3f3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2ca725f8269c441bb985336decc9d57e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2e680e7a08ec4eb1adfdd8f1ea37a423] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=46.9 K 2024-11-19T21:41:03,197 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e6c13d419e647dcafa4d0d1f3f57cf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732052460965 2024-11-19T21:41:03,197 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d666cad349a451c832bef70157da3f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732052461007 2024-11-19T21:41:03,198 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ca725f8269c441bb985336decc9d57e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732052461656 2024-11-19T21:41:03,198 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e680e7a08ec4eb1adfdd8f1ea37a423, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732052462334 2024-11-19T21:41:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T21:41:03,217 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#231 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:03,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/a7ba88f41e354a2e9e4e818176eca1e6 is 50, key is test_row_0/C:col10/1732052462335/Put/seqid=0 2024-11-19T21:41:03,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742102_1278 (size=12139) 2024-11-19T21:41:03,257 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/a7ba88f41e354a2e9e4e818176eca1e6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a7ba88f41e354a2e9e4e818176eca1e6 2024-11-19T21:41:03,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T21:41:03,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:03,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,269 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into a7ba88f41e354a2e9e4e818176eca1e6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:03,269 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:03,269 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=12, startTime=1732052463093; duration=0sec 2024-11-19T21:41:03,269 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:03,269 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:03,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052523298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052523299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052523299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052523299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052523299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T21:41:03,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T21:41:03,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052523508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052523508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052523508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052523508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052523508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/79f2294723b3443a85a8b74a385d730f 2024-11-19T21:41:03,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/3dfc7e64d2ee405d85fe3fc772890441 is 50, key is test_row_0/B:col10/1732052462980/Put/seqid=0 2024-11-19T21:41:03,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T21:41:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,575 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7d279bb86c5d499c9932f805a796e1fd as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7d279bb86c5d499c9932f805a796e1fd 2024-11-19T21:41:03,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742103_1279 (size=12001) 2024-11-19T21:41:03,580 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into 7d279bb86c5d499c9932f805a796e1fd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:03,580 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:03,580 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=12, startTime=1732052463093; duration=0sec 2024-11-19T21:41:03,580 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:03,580 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:03,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T21:41:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T21:41:03,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052523818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052523818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052523818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052523818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052523818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:03,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T21:41:03,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:03,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:03,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:03,940 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T21:41:03,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/3dfc7e64d2ee405d85fe3fc772890441 2024-11-19T21:41:03,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0c63de8912a546afb3a9046d731063bf is 50, key is test_row_0/C:col10/1732052462980/Put/seqid=0 2024-11-19T21:41:03,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742104_1280 (size=12001) 2024-11-19T21:41:03,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0c63de8912a546afb3a9046d731063bf 2024-11-19T21:41:03,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/79f2294723b3443a85a8b74a385d730f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/79f2294723b3443a85a8b74a385d730f 2024-11-19T21:41:04,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/79f2294723b3443a85a8b74a385d730f, entries=200, sequenceid=90, filesize=14.0 K 2024-11-19T21:41:04,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/3dfc7e64d2ee405d85fe3fc772890441 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/3dfc7e64d2ee405d85fe3fc772890441 2024-11-19T21:41:04,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/3dfc7e64d2ee405d85fe3fc772890441, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T21:41:04,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0c63de8912a546afb3a9046d731063bf as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0c63de8912a546afb3a9046d731063bf 2024-11-19T21:41:04,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0c63de8912a546afb3a9046d731063bf, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T21:41:04,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d5c69b89f3a83a34d8bee2d0fed83b40 in 921ms, sequenceid=90, compaction requested=false 2024-11-19T21:41:04,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:04,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T21:41:04,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:04,025 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:41:04,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/1c4ae2c3fddf4ff0a3b4f767625d2aae is 50, key is test_row_0/A:col10/1732052463188/Put/seqid=0 2024-11-19T21:41:04,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742105_1281 (size=12001) 2024-11-19T21:41:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T21:41:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:04,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:04,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052524335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052524336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052524336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052524337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052524338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052524448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052524448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052524448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052524448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052524448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,453 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/1c4ae2c3fddf4ff0a3b4f767625d2aae 2024-11-19T21:41:04,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/38694a03679e444bba93ceee157e3f15 is 50, key is test_row_0/B:col10/1732052463188/Put/seqid=0 2024-11-19T21:41:04,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742106_1282 (size=12001) 2024-11-19T21:41:04,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052524658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052524658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052524658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052524659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052524659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,883 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/38694a03679e444bba93ceee157e3f15 2024-11-19T21:41:04,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/fb3d1934ce74442d9a8557c6b38c47fb is 50, key is test_row_0/C:col10/1732052463188/Put/seqid=0 2024-11-19T21:41:04,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742107_1283 (size=12001) 2024-11-19T21:41:04,894 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/fb3d1934ce74442d9a8557c6b38c47fb 2024-11-19T21:41:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/1c4ae2c3fddf4ff0a3b4f767625d2aae as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1c4ae2c3fddf4ff0a3b4f767625d2aae 2024-11-19T21:41:04,903 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1c4ae2c3fddf4ff0a3b4f767625d2aae, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T21:41:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/38694a03679e444bba93ceee157e3f15 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/38694a03679e444bba93ceee157e3f15 2024-11-19T21:41:04,907 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/38694a03679e444bba93ceee157e3f15, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T21:41:04,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/fb3d1934ce74442d9a8557c6b38c47fb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fb3d1934ce74442d9a8557c6b38c47fb 2024-11-19T21:41:04,912 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fb3d1934ce74442d9a8557c6b38c47fb, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T21:41:04,913 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d5c69b89f3a83a34d8bee2d0fed83b40 in 888ms, sequenceid=116, compaction requested=true 2024-11-19T21:41:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T21:41:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T21:41:04,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T21:41:04,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8110 sec 2024-11-19T21:41:04,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8170 sec 2024-11-19T21:41:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:04,971 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:04,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T21:41:04,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/e6941b868ff1412288cab18a8892df44 is 50, key is test_row_0/A:col10/1732052464972/Put/seqid=0 2024-11-19T21:41:04,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742108_1284 (size=16931) 2024-11-19T21:41:04,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052524988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052524990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052524991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052524991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052524993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052525098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052525098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052525099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052525099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052525108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T21:41:05,248 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:05,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T21:41:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T21:41:05,250 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:05,251 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:05,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052525308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052525309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052525309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052525309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052525318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T21:41:05,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/e6941b868ff1412288cab18a8892df44 2024-11-19T21:41:05,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/e1ca06f5835047ff8070d46cd07b7afe is 50, key is test_row_0/B:col10/1732052464972/Put/seqid=0 2024-11-19T21:41:05,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742109_1285 (size=12151) 2024-11-19T21:41:05,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T21:41:05,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:05,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:05,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:05,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T21:41:05,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:05,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:05,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T21:41:05,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052525617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052525618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052525618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052525618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052525628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:05,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T21:41:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:05,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/e1ca06f5835047ff8070d46cd07b7afe 2024-11-19T21:41:05,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/8a6e8ed500e04bdd8c8ccb924f1fdae7 is 50, key is test_row_0/C:col10/1732052464972/Put/seqid=0 2024-11-19T21:41:05,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742110_1286 (size=12151) 2024-11-19T21:41:05,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/8a6e8ed500e04bdd8c8ccb924f1fdae7 2024-11-19T21:41:05,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/e6941b868ff1412288cab18a8892df44 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/e6941b868ff1412288cab18a8892df44 2024-11-19T21:41:05,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/e6941b868ff1412288cab18a8892df44, entries=250, sequenceid=134, filesize=16.5 K 2024-11-19T21:41:05,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/e1ca06f5835047ff8070d46cd07b7afe as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e1ca06f5835047ff8070d46cd07b7afe 2024-11-19T21:41:05,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e1ca06f5835047ff8070d46cd07b7afe, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T21:41:05,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/8a6e8ed500e04bdd8c8ccb924f1fdae7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8a6e8ed500e04bdd8c8ccb924f1fdae7 2024-11-19T21:41:05,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8a6e8ed500e04bdd8c8ccb924f1fdae7, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T21:41:05,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d5c69b89f3a83a34d8bee2d0fed83b40 in 870ms, sequenceid=134, compaction requested=true 2024-11-19T21:41:05,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:05,841 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:41:05,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:05,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:05,842 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:41:05,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:05,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:05,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:05,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:05,843 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55412 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:41:05,843 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:05,843 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,843 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/b4ea200ddffc469abf4a1b7d371f6845, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/79f2294723b3443a85a8b74a385d730f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1c4ae2c3fddf4ff0a3b4f767625d2aae, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/e6941b868ff1412288cab18a8892df44] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=54.1 K 2024-11-19T21:41:05,844 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:41:05,844 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:05,844 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,844 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7d279bb86c5d499c9932f805a796e1fd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/3dfc7e64d2ee405d85fe3fc772890441, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/38694a03679e444bba93ceee157e3f15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e1ca06f5835047ff8070d46cd07b7afe] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=47.2 K 2024-11-19T21:41:05,845 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d279bb86c5d499c9932f805a796e1fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732052462334 2024-11-19T21:41:05,845 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4ea200ddffc469abf4a1b7d371f6845, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732052462334 2024-11-19T21:41:05,846 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dfc7e64d2ee405d85fe3fc772890441, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732052462979 2024-11-19T21:41:05,846 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f2294723b3443a85a8b74a385d730f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732052462979 2024-11-19T21:41:05,846 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 38694a03679e444bba93ceee157e3f15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732052463131 2024-11-19T21:41:05,846 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c4ae2c3fddf4ff0a3b4f767625d2aae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732052463131 2024-11-19T21:41:05,847 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e1ca06f5835047ff8070d46cd07b7afe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732052464970 2024-11-19T21:41:05,847 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6941b868ff1412288cab18a8892df44, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732052464970 2024-11-19T21:41:05,861 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:05,861 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/f0ea9d7815e7484987f675515c64b4d7 is 50, key is test_row_0/B:col10/1732052464972/Put/seqid=0 2024-11-19T21:41:05,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T21:41:05,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,862 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:05,862 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T21:41:05,864 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/23c5e4c4e6454654a6f197bf04338c32 is 50, key is test_row_0/A:col10/1732052464972/Put/seqid=0 2024-11-19T21:41:05,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/1bae88889f9446e0b5fe9337cb47ca9a is 50, key is test_row_0/A:col10/1732052464991/Put/seqid=0 2024-11-19T21:41:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T21:41:05,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742111_1287 (size=12425) 2024-11-19T21:41:05,891 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/f0ea9d7815e7484987f675515c64b4d7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f0ea9d7815e7484987f675515c64b4d7 2024-11-19T21:41:05,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742112_1288 (size=12425) 2024-11-19T21:41:05,897 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into f0ea9d7815e7484987f675515c64b4d7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:05,897 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:05,897 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=12, startTime=1732052465842; duration=0sec 2024-11-19T21:41:05,897 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:05,897 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:05,897 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:41:05,901 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/23c5e4c4e6454654a6f197bf04338c32 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/23c5e4c4e6454654a6f197bf04338c32 2024-11-19T21:41:05,901 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:41:05,901 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:05,901 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:05,901 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a7ba88f41e354a2e9e4e818176eca1e6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0c63de8912a546afb3a9046d731063bf, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fb3d1934ce74442d9a8557c6b38c47fb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8a6e8ed500e04bdd8c8ccb924f1fdae7] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=47.2 K 2024-11-19T21:41:05,901 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a7ba88f41e354a2e9e4e818176eca1e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732052462334 2024-11-19T21:41:05,902 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c63de8912a546afb3a9046d731063bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732052462979 2024-11-19T21:41:05,902 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting fb3d1934ce74442d9a8557c6b38c47fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732052463131 2024-11-19T21:41:05,903 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6e8ed500e04bdd8c8ccb924f1fdae7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732052464970 2024-11-19T21:41:05,908 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into 23c5e4c4e6454654a6f197bf04338c32(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:05,908 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:05,908 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=12, startTime=1732052465841; duration=0sec 2024-11-19T21:41:05,908 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:05,908 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:05,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742113_1289 (size=12151) 2024-11-19T21:41:05,912 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/1bae88889f9446e0b5fe9337cb47ca9a 2024-11-19T21:41:05,920 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#243 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:05,920 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/65d94c898c9d41559053dec0e0a2a2ba is 50, key is test_row_0/C:col10/1732052464972/Put/seqid=0 2024-11-19T21:41:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/414e707967984abc81245ac158aac613 is 50, key is test_row_0/B:col10/1732052464991/Put/seqid=0 2024-11-19T21:41:05,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742114_1290 (size=12425) 2024-11-19T21:41:05,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742115_1291 (size=12151) 2024-11-19T21:41:05,957 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/414e707967984abc81245ac158aac613 2024-11-19T21:41:05,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/b878c34115b848b381efef239a96d6ce is 50, key is test_row_0/C:col10/1732052464991/Put/seqid=0 2024-11-19T21:41:05,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742116_1292 (size=12151) 2024-11-19T21:41:05,972 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/b878c34115b848b381efef239a96d6ce 2024-11-19T21:41:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/1bae88889f9446e0b5fe9337cb47ca9a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1bae88889f9446e0b5fe9337cb47ca9a 2024-11-19T21:41:05,982 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1bae88889f9446e0b5fe9337cb47ca9a, entries=150, sequenceid=152, filesize=11.9 K 2024-11-19T21:41:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/414e707967984abc81245ac158aac613 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/414e707967984abc81245ac158aac613 2024-11-19T21:41:05,989 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/414e707967984abc81245ac158aac613, entries=150, sequenceid=152, filesize=11.9 K 2024-11-19T21:41:05,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/b878c34115b848b381efef239a96d6ce as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b878c34115b848b381efef239a96d6ce 2024-11-19T21:41:05,998 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b878c34115b848b381efef239a96d6ce, entries=150, sequenceid=152, filesize=11.9 K 2024-11-19T21:41:05,999 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for d5c69b89f3a83a34d8bee2d0fed83b40 in 137ms, sequenceid=152, compaction requested=false 2024-11-19T21:41:05,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:05,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:06,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T21:41:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T21:41:06,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T21:41:06,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 751 msec 2024-11-19T21:41:06,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 757 msec 2024-11-19T21:41:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:06,136 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:06,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:41:06,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/a16e909541204b67bbeea473e4764893 is 50, key is test_row_0/A:col10/1732052466134/Put/seqid=0 2024-11-19T21:41:06,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742117_1293 (size=12147) 2024-11-19T21:41:06,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052526200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052526201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052526203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052526204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052526204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052526308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052526308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052526318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052526318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052526319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,349 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/65d94c898c9d41559053dec0e0a2a2ba as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/65d94c898c9d41559053dec0e0a2a2ba 2024-11-19T21:41:06,353 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into 65d94c898c9d41559053dec0e0a2a2ba(size=12.1 K), total size for store is 24 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:06,353 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:06,353 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=12, startTime=1732052465842; duration=0sec 2024-11-19T21:41:06,353 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:06,353 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T21:41:06,389 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:06,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T21:41:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T21:41:06,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:06,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:06,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T21:41:06,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052526518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052526518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052526528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052526528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052526528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T21:41:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:06,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:06,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/a16e909541204b67bbeea473e4764893 2024-11-19T21:41:06,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/09c98888c7f347afac9d9180e36d80af is 50, key is test_row_0/B:col10/1732052466134/Put/seqid=0 2024-11-19T21:41:06,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742118_1294 (size=9757) 2024-11-19T21:41:06,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T21:41:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:06,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T21:41:06,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052526828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052526828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052526838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052526838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052526838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:06,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T21:41:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:06,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:06,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/09c98888c7f347afac9d9180e36d80af 2024-11-19T21:41:06,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/3e99a26441dd4b3fb53feb3a305fd8ba is 50, key is test_row_0/C:col10/1732052466134/Put/seqid=0 2024-11-19T21:41:06,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742119_1295 (size=9757) 2024-11-19T21:41:07,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T21:41:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T21:41:07,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T21:41:07,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:07,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:07,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T21:41:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:07,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052527338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:07,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052527338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:07,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052527348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:07,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052527348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:07,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052527348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:07,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/3e99a26441dd4b3fb53feb3a305fd8ba 2024-11-19T21:41:07,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/a16e909541204b67bbeea473e4764893 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a16e909541204b67bbeea473e4764893 2024-11-19T21:41:07,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a16e909541204b67bbeea473e4764893, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T21:41:07,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/09c98888c7f347afac9d9180e36d80af as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/09c98888c7f347afac9d9180e36d80af 2024-11-19T21:41:07,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/09c98888c7f347afac9d9180e36d80af, entries=100, sequenceid=166, filesize=9.5 K 2024-11-19T21:41:07,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/3e99a26441dd4b3fb53feb3a305fd8ba as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/3e99a26441dd4b3fb53feb3a305fd8ba 2024-11-19T21:41:07,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/3e99a26441dd4b3fb53feb3a305fd8ba, entries=100, sequenceid=166, filesize=9.5 K 2024-11-19T21:41:07,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d5c69b89f3a83a34d8bee2d0fed83b40 in 1272ms, sequenceid=166, compaction requested=true 2024-11-19T21:41:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:07,408 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:07,408 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:07,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:07,409 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:07,409 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36723 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:07,409 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:07,409 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:07,409 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,409 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,409 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f0ea9d7815e7484987f675515c64b4d7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/414e707967984abc81245ac158aac613, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/09c98888c7f347afac9d9180e36d80af] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=33.5 K 2024-11-19T21:41:07,409 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/23c5e4c4e6454654a6f197bf04338c32, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1bae88889f9446e0b5fe9337cb47ca9a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a16e909541204b67bbeea473e4764893] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=35.9 K 2024-11-19T21:41:07,409 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting f0ea9d7815e7484987f675515c64b4d7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732052464970 2024-11-19T21:41:07,410 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23c5e4c4e6454654a6f197bf04338c32, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732052464970 2024-11-19T21:41:07,410 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 414e707967984abc81245ac158aac613, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732052464990 2024-11-19T21:41:07,410 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bae88889f9446e0b5fe9337cb47ca9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732052464990 2024-11-19T21:41:07,410 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting a16e909541204b67bbeea473e4764893, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732052466134 2024-11-19T21:41:07,410 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c98888c7f347afac9d9180e36d80af, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732052466134 2024-11-19T21:41:07,416 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:07,416 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/cfe6741370434b8bac99b241e08fc376 is 50, key is test_row_0/B:col10/1732052466134/Put/seqid=0 2024-11-19T21:41:07,416 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:07,417 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/d82bc4943b2148148451cfe8758dce61 is 50, key is test_row_0/A:col10/1732052466134/Put/seqid=0 2024-11-19T21:41:07,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742120_1296 (size=12527) 2024-11-19T21:41:07,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742121_1297 (size=12527) 2024-11-19T21:41:07,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T21:41:07,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,467 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:41:07,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/89cc08e367db4b30bddc2b6ccd57da56 is 50, key is test_row_0/A:col10/1732052466203/Put/seqid=0 2024-11-19T21:41:07,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742122_1298 (size=12151) 2024-11-19T21:41:07,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T21:41:07,827 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/cfe6741370434b8bac99b241e08fc376 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/cfe6741370434b8bac99b241e08fc376 2024-11-19T21:41:07,835 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into cfe6741370434b8bac99b241e08fc376(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:07,835 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:07,835 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=13, startTime=1732052467408; duration=0sec 2024-11-19T21:41:07,836 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:07,836 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:07,836 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:07,837 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:07,838 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:07,838 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,838 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/65d94c898c9d41559053dec0e0a2a2ba, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b878c34115b848b381efef239a96d6ce, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/3e99a26441dd4b3fb53feb3a305fd8ba] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=33.5 K 2024-11-19T21:41:07,838 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d94c898c9d41559053dec0e0a2a2ba, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732052464970 2024-11-19T21:41:07,839 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b878c34115b848b381efef239a96d6ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732052464990 2024-11-19T21:41:07,839 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e99a26441dd4b3fb53feb3a305fd8ba, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732052466134 2024-11-19T21:41:07,841 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/d82bc4943b2148148451cfe8758dce61 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d82bc4943b2148148451cfe8758dce61 2024-11-19T21:41:07,846 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into d82bc4943b2148148451cfe8758dce61(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:07,846 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:07,847 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=13, startTime=1732052467408; duration=0sec 2024-11-19T21:41:07,847 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:07,847 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:07,850 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:07,850 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/90fa0072684a4a3487e737624b092165 is 50, key is test_row_0/C:col10/1732052466134/Put/seqid=0 2024-11-19T21:41:07,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742123_1299 (size=12527) 2024-11-19T21:41:07,862 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/90fa0072684a4a3487e737624b092165 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/90fa0072684a4a3487e737624b092165 2024-11-19T21:41:07,868 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into 90fa0072684a4a3487e737624b092165(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:07,868 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:07,868 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=13, startTime=1732052467408; duration=0sec 2024-11-19T21:41:07,868 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:07,868 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:07,876 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/89cc08e367db4b30bddc2b6ccd57da56 2024-11-19T21:41:07,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/af0fcb8de1aa40878698be533547f1d5 is 50, key is test_row_0/B:col10/1732052466203/Put/seqid=0 2024-11-19T21:41:07,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742124_1300 (size=12151) 2024-11-19T21:41:07,901 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/af0fcb8de1aa40878698be533547f1d5 2024-11-19T21:41:07,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0046f43fa29c4e83ada0ff7cf057df4d is 50, key is test_row_0/C:col10/1732052466203/Put/seqid=0 2024-11-19T21:41:07,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742125_1301 (size=12151) 2024-11-19T21:41:07,925 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0046f43fa29c4e83ada0ff7cf057df4d 2024-11-19T21:41:07,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/89cc08e367db4b30bddc2b6ccd57da56 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/89cc08e367db4b30bddc2b6ccd57da56 2024-11-19T21:41:07,934 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/89cc08e367db4b30bddc2b6ccd57da56, entries=150, sequenceid=191, filesize=11.9 K 2024-11-19T21:41:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/af0fcb8de1aa40878698be533547f1d5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/af0fcb8de1aa40878698be533547f1d5 2024-11-19T21:41:07,943 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/af0fcb8de1aa40878698be533547f1d5, entries=150, sequenceid=191, filesize=11.9 K 2024-11-19T21:41:07,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0046f43fa29c4e83ada0ff7cf057df4d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0046f43fa29c4e83ada0ff7cf057df4d 2024-11-19T21:41:07,952 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0046f43fa29c4e83ada0ff7cf057df4d, entries=150, sequenceid=191, filesize=11.9 K 2024-11-19T21:41:07,952 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for d5c69b89f3a83a34d8bee2d0fed83b40 in 485ms, sequenceid=191, compaction requested=false 2024-11-19T21:41:07,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:07,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:07,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T21:41:07,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T21:41:07,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T21:41:07,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5580 sec 2024-11-19T21:41:07,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.5680 sec 2024-11-19T21:41:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:08,357 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:08,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:41:08,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/01845cae3f074f419b730c3b74735eac is 50, key is test_row_0/A:col10/1732052468354/Put/seqid=0 2024-11-19T21:41:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052528387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052528386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052528390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052528390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052528394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742126_1302 (size=14541) 2024-11-19T21:41:08,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/01845cae3f074f419b730c3b74735eac 2024-11-19T21:41:08,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/b9dac01325224bf1b9aebc3696947936 is 50, key is test_row_0/B:col10/1732052468354/Put/seqid=0 2024-11-19T21:41:08,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742127_1303 (size=12151) 2024-11-19T21:41:08,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/b9dac01325224bf1b9aebc3696947936 2024-11-19T21:41:08,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/d3db249bbfd747708e8f7bdc452f68f1 is 50, key is test_row_0/C:col10/1732052468354/Put/seqid=0 2024-11-19T21:41:08,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742128_1304 (size=12151) 2024-11-19T21:41:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052528499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052528499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052528499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052528508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052528508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T21:41:08,539 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:08,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T21:41:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T21:41:08,547 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:08,548 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:08,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T21:41:08,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T21:41:08,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:08,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:08,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:08,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:08,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:08,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052528708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052528708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052528708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052528718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052528718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:08,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/d3db249bbfd747708e8f7bdc452f68f1 2024-11-19T21:41:08,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T21:41:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:08,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:08,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/01845cae3f074f419b730c3b74735eac as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/01845cae3f074f419b730c3b74735eac 2024-11-19T21:41:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:08,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/01845cae3f074f419b730c3b74735eac, entries=200, sequenceid=205, filesize=14.2 K 2024-11-19T21:41:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T21:41:08,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/b9dac01325224bf1b9aebc3696947936 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/b9dac01325224bf1b9aebc3696947936 2024-11-19T21:41:08,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/b9dac01325224bf1b9aebc3696947936, entries=150, sequenceid=205, filesize=11.9 K 2024-11-19T21:41:08,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/d3db249bbfd747708e8f7bdc452f68f1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d3db249bbfd747708e8f7bdc452f68f1 2024-11-19T21:41:08,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d3db249bbfd747708e8f7bdc452f68f1, entries=150, sequenceid=205, filesize=11.9 K 2024-11-19T21:41:08,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d5c69b89f3a83a34d8bee2d0fed83b40 in 509ms, sequenceid=205, compaction requested=true 2024-11-19T21:41:08,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:08,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:08,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:08,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:08,866 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:08,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:08,867 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:08,867 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:08,867 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:08,867 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:08,867 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:08,867 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:08,868 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d82bc4943b2148148451cfe8758dce61, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/89cc08e367db4b30bddc2b6ccd57da56, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/01845cae3f074f419b730c3b74735eac] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=38.3 K 2024-11-19T21:41:08,868 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:08,868 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/cfe6741370434b8bac99b241e08fc376, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/af0fcb8de1aa40878698be533547f1d5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/b9dac01325224bf1b9aebc3696947936] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.0 K 2024-11-19T21:41:08,868 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting d82bc4943b2148148451cfe8758dce61, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732052464990 2024-11-19T21:41:08,868 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting cfe6741370434b8bac99b241e08fc376, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732052464990 2024-11-19T21:41:08,868 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89cc08e367db4b30bddc2b6ccd57da56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732052466194 2024-11-19T21:41:08,868 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting af0fcb8de1aa40878698be533547f1d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732052466194 2024-11-19T21:41:08,869 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01845cae3f074f419b730c3b74735eac, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732052468352 2024-11-19T21:41:08,869 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b9dac01325224bf1b9aebc3696947936, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732052468354 2024-11-19T21:41:08,874 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:08,874 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:08,875 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/ee6bb7c92a54471983481e3b6cd28542 is 50, key is test_row_0/B:col10/1732052468354/Put/seqid=0 2024-11-19T21:41:08,875 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/1a7bfb0656c94c8fb701b1b137c98d0f is 50, key is test_row_0/A:col10/1732052468354/Put/seqid=0 2024-11-19T21:41:08,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742130_1306 (size=12629) 2024-11-19T21:41:08,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742129_1305 (size=12629) 2024-11-19T21:41:08,882 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/1a7bfb0656c94c8fb701b1b137c98d0f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1a7bfb0656c94c8fb701b1b137c98d0f 2024-11-19T21:41:08,886 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into 1a7bfb0656c94c8fb701b1b137c98d0f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:08,886 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:08,886 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=13, startTime=1732052468866; duration=0sec 2024-11-19T21:41:08,886 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:08,886 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:08,886 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:08,887 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:08,887 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:08,887 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:08,887 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/90fa0072684a4a3487e737624b092165, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0046f43fa29c4e83ada0ff7cf057df4d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d3db249bbfd747708e8f7bdc452f68f1] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.0 K 2024-11-19T21:41:08,888 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90fa0072684a4a3487e737624b092165, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732052464990 2024-11-19T21:41:08,888 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0046f43fa29c4e83ada0ff7cf057df4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732052466194 2024-11-19T21:41:08,888 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3db249bbfd747708e8f7bdc452f68f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732052468354 2024-11-19T21:41:08,894 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:08,895 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/331105c72a3e4554a29e06801b5f628e is 50, key is test_row_0/C:col10/1732052468354/Put/seqid=0 2024-11-19T21:41:08,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742131_1307 (size=12629) 2024-11-19T21:41:09,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T21:41:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:09,006 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:41:09,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/819e0da8d11a4b90a453bc03c4765f57 is 50, key is test_row_0/A:col10/1732052468379/Put/seqid=0 2024-11-19T21:41:09,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742132_1308 (size=12151) 2024-11-19T21:41:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:09,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:09,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052529025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052529028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052529028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052529029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052529029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052529138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052529138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052529138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T21:41:09,283 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/ee6bb7c92a54471983481e3b6cd28542 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ee6bb7c92a54471983481e3b6cd28542 2024-11-19T21:41:09,288 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into ee6bb7c92a54471983481e3b6cd28542(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:09,288 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:09,288 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=13, startTime=1732052468866; duration=0sec 2024-11-19T21:41:09,288 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:09,288 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:09,316 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/331105c72a3e4554a29e06801b5f628e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/331105c72a3e4554a29e06801b5f628e 2024-11-19T21:41:09,321 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into 331105c72a3e4554a29e06801b5f628e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:09,321 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:09,321 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=13, startTime=1732052468867; duration=0sec 2024-11-19T21:41:09,321 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:09,321 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:09,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052529348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052529348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052529348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,415 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/819e0da8d11a4b90a453bc03c4765f57 2024-11-19T21:41:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/0f8fe8ebe267471daebb364e34afa99d is 50, key is test_row_0/B:col10/1732052468379/Put/seqid=0 2024-11-19T21:41:09,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742133_1309 (size=12151) 2024-11-19T21:41:09,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052529538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052529538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052529658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052529658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052529658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T21:41:09,825 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/0f8fe8ebe267471daebb364e34afa99d 2024-11-19T21:41:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/cf89669aa116484cbbf9567bc2b17f09 is 50, key is test_row_0/C:col10/1732052468379/Put/seqid=0 2024-11-19T21:41:09,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742134_1310 (size=12151) 2024-11-19T21:41:10,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052530168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:10,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052530168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052530168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:10,241 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/cf89669aa116484cbbf9567bc2b17f09 2024-11-19T21:41:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/819e0da8d11a4b90a453bc03c4765f57 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/819e0da8d11a4b90a453bc03c4765f57 2024-11-19T21:41:10,248 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/819e0da8d11a4b90a453bc03c4765f57, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T21:41:10,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/0f8fe8ebe267471daebb364e34afa99d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/0f8fe8ebe267471daebb364e34afa99d 2024-11-19T21:41:10,252 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/0f8fe8ebe267471daebb364e34afa99d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T21:41:10,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/cf89669aa116484cbbf9567bc2b17f09 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/cf89669aa116484cbbf9567bc2b17f09 2024-11-19T21:41:10,256 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/cf89669aa116484cbbf9567bc2b17f09, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T21:41:10,257 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d5c69b89f3a83a34d8bee2d0fed83b40 in 1251ms, sequenceid=231, compaction requested=false 2024-11-19T21:41:10,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:10,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:10,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T21:41:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T21:41:10,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T21:41:10,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7100 sec 2024-11-19T21:41:10,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7210 sec 2024-11-19T21:41:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:10,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:10,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:41:10,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/c30b6e25b5684879b026d1b5d76facb2 is 50, key is test_row_0/A:col10/1732052469028/Put/seqid=0 2024-11-19T21:41:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742135_1311 (size=12151) 2024-11-19T21:41:10,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052530590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:10,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052530592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T21:41:10,688 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:10,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T21:41:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T21:41:10,690 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:10,691 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:10,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:10,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052530698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:10,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052530708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T21:41:10,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T21:41:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:10,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:10,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052530908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:10,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052530918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:10,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/c30b6e25b5684879b026d1b5d76facb2 2024-11-19T21:41:10,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/25d66a0803554ac4b59ddd6c0347f045 is 50, key is test_row_0/B:col10/1732052469028/Put/seqid=0 2024-11-19T21:41:10,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742136_1312 (size=12151) 2024-11-19T21:41:10,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T21:41:10,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:10,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:10,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:10,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:10,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T21:41:11,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T21:41:11,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:11,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052531178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:11,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052531178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:11,179 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:11,179 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:11,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:11,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052531188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:11,190 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:11,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052531218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:11,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:11,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052531228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:11,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T21:41:11,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:11,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T21:41:11,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/25d66a0803554ac4b59ddd6c0347f045 2024-11-19T21:41:11,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/a96335f9bf2c4297be51d864a830266a is 50, key is test_row_0/C:col10/1732052469028/Put/seqid=0 2024-11-19T21:41:11,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742137_1313 (size=12151) 2024-11-19T21:41:11,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T21:41:11,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:11,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T21:41:11,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:11,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052531728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:11,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052531738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:11,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T21:41:11,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:11,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:11,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/a96335f9bf2c4297be51d864a830266a 2024-11-19T21:41:11,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/c30b6e25b5684879b026d1b5d76facb2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c30b6e25b5684879b026d1b5d76facb2 2024-11-19T21:41:11,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c30b6e25b5684879b026d1b5d76facb2, entries=150, sequenceid=245, filesize=11.9 K 2024-11-19T21:41:11,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/25d66a0803554ac4b59ddd6c0347f045 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/25d66a0803554ac4b59ddd6c0347f045 2024-11-19T21:41:11,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/25d66a0803554ac4b59ddd6c0347f045, entries=150, sequenceid=245, filesize=11.9 K 2024-11-19T21:41:11,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/a96335f9bf2c4297be51d864a830266a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a96335f9bf2c4297be51d864a830266a 2024-11-19T21:41:11,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a96335f9bf2c4297be51d864a830266a, entries=150, sequenceid=245, filesize=11.9 K 2024-11-19T21:41:11,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d5c69b89f3a83a34d8bee2d0fed83b40 in 1249ms, sequenceid=245, compaction requested=true 2024-11-19T21:41:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:11,800 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:11,800 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:11,801 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:11,801 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:11,801 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:11,801 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:11,801 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,801 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,801 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ee6bb7c92a54471983481e3b6cd28542, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/0f8fe8ebe267471daebb364e34afa99d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/25d66a0803554ac4b59ddd6c0347f045] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.1 K 2024-11-19T21:41:11,801 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1a7bfb0656c94c8fb701b1b137c98d0f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/819e0da8d11a4b90a453bc03c4765f57, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c30b6e25b5684879b026d1b5d76facb2] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.1 K 2024-11-19T21:41:11,801 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a7bfb0656c94c8fb701b1b137c98d0f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732052468354 2024-11-19T21:41:11,801 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting ee6bb7c92a54471983481e3b6cd28542, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732052468354 2024-11-19T21:41:11,801 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 819e0da8d11a4b90a453bc03c4765f57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732052468379 2024-11-19T21:41:11,801 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8fe8ebe267471daebb364e34afa99d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732052468379 2024-11-19T21:41:11,802 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d66a0803554ac4b59ddd6c0347f045, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732052469024 2024-11-19T21:41:11,802 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30b6e25b5684879b026d1b5d76facb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732052469024 2024-11-19T21:41:11,808 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:11,809 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:11,809 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/01b91f291e1849a3aea33c9ac156fe4c is 50, key is test_row_0/B:col10/1732052469028/Put/seqid=0 2024-11-19T21:41:11,809 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f198f009a7554fb4ab0291395628c164 is 50, key is test_row_0/A:col10/1732052469028/Put/seqid=0 2024-11-19T21:41:11,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742138_1314 (size=12731) 2024-11-19T21:41:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T21:41:11,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742139_1315 (size=12731) 2024-11-19T21:41:11,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T21:41:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:11,913 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:41:11,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/99931e6f1bf84b49a88d94f4dc195f0b is 50, key is test_row_0/A:col10/1732052470582/Put/seqid=0 2024-11-19T21:41:11,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742140_1316 (size=12301) 2024-11-19T21:41:12,228 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/01b91f291e1849a3aea33c9ac156fe4c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/01b91f291e1849a3aea33c9ac156fe4c 2024-11-19T21:41:12,236 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into 01b91f291e1849a3aea33c9ac156fe4c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:12,236 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f198f009a7554fb4ab0291395628c164 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f198f009a7554fb4ab0291395628c164 2024-11-19T21:41:12,236 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:12,236 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=13, startTime=1732052471800; duration=0sec 2024-11-19T21:41:12,237 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:12,237 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:12,237 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:12,238 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:12,238 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:12,238 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:12,238 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/331105c72a3e4554a29e06801b5f628e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/cf89669aa116484cbbf9567bc2b17f09, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a96335f9bf2c4297be51d864a830266a] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.1 K 2024-11-19T21:41:12,238 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 331105c72a3e4554a29e06801b5f628e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732052468354 2024-11-19T21:41:12,238 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting cf89669aa116484cbbf9567bc2b17f09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732052468379 2024-11-19T21:41:12,239 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a96335f9bf2c4297be51d864a830266a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732052469024 2024-11-19T21:41:12,240 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into f198f009a7554fb4ab0291395628c164(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:12,240 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:12,240 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=13, startTime=1732052471800; duration=0sec 2024-11-19T21:41:12,241 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:12,241 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:12,245 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:12,245 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/398ab7fef2b94d39adbe952b40b85a15 is 50, key is test_row_0/C:col10/1732052469028/Put/seqid=0 2024-11-19T21:41:12,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742141_1317 (size=12731) 2024-11-19T21:41:12,322 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/99931e6f1bf84b49a88d94f4dc195f0b 2024-11-19T21:41:12,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/2e29aed923874702a1ffe4fa3bdb0ee7 is 50, key is test_row_0/B:col10/1732052470582/Put/seqid=0 2024-11-19T21:41:12,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742142_1318 (size=12301) 2024-11-19T21:41:12,663 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/398ab7fef2b94d39adbe952b40b85a15 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/398ab7fef2b94d39adbe952b40b85a15 2024-11-19T21:41:12,668 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into 398ab7fef2b94d39adbe952b40b85a15(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:12,668 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:12,668 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=13, startTime=1732052471800; duration=0sec 2024-11-19T21:41:12,669 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:12,669 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:12,758 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/2e29aed923874702a1ffe4fa3bdb0ee7 2024-11-19T21:41:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:12,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/8ce40b590ab84bf79185acbae79cedee is 50, key is test_row_0/C:col10/1732052470582/Put/seqid=0 2024-11-19T21:41:12,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742143_1319 (size=12301) 2024-11-19T21:41:12,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052532779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:12,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052532781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T21:41:12,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052532888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:12,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052532888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052533098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052533098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,173 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/8ce40b590ab84bf79185acbae79cedee 2024-11-19T21:41:13,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/99931e6f1bf84b49a88d94f4dc195f0b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/99931e6f1bf84b49a88d94f4dc195f0b 2024-11-19T21:41:13,185 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/99931e6f1bf84b49a88d94f4dc195f0b, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T21:41:13,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/2e29aed923874702a1ffe4fa3bdb0ee7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2e29aed923874702a1ffe4fa3bdb0ee7 2024-11-19T21:41:13,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052533188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,191 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,192 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2e29aed923874702a1ffe4fa3bdb0ee7, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T21:41:13,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/8ce40b590ab84bf79185acbae79cedee as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8ce40b590ab84bf79185acbae79cedee 2024-11-19T21:41:13,198 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8ce40b590ab84bf79185acbae79cedee, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T21:41:13,199 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for d5c69b89f3a83a34d8bee2d0fed83b40 in 1286ms, sequenceid=269, compaction requested=false 2024-11-19T21:41:13,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:13,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:13,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T21:41:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T21:41:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:13,202 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:13,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T21:41:13,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T21:41:13,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5100 sec 2024-11-19T21:41:13,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/d190636a575d4e57ae28bf895b9ec333 is 50, key is test_row_0/A:col10/1732052473201/Put/seqid=0 2024-11-19T21:41:13,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.5190 sec 2024-11-19T21:41:13,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742144_1320 (size=12301) 2024-11-19T21:41:13,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/d190636a575d4e57ae28bf895b9ec333 2024-11-19T21:41:13,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/f921280b1c3745069db91df1d359a245 is 50, key is test_row_0/B:col10/1732052473201/Put/seqid=0 2024-11-19T21:41:13,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742145_1321 (size=12301) 2024-11-19T21:41:13,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/f921280b1c3745069db91df1d359a245 2024-11-19T21:41:13,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/acb50c7007344ff3b738211eee8fa73f is 50, key is test_row_0/C:col10/1732052473201/Put/seqid=0 2024-11-19T21:41:13,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742146_1322 (size=12301) 2024-11-19T21:41:13,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/acb50c7007344ff3b738211eee8fa73f 2024-11-19T21:41:13,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052533248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/d190636a575d4e57ae28bf895b9ec333 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d190636a575d4e57ae28bf895b9ec333 2024-11-19T21:41:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052533252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d190636a575d4e57ae28bf895b9ec333, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T21:41:13,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/f921280b1c3745069db91df1d359a245 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f921280b1c3745069db91df1d359a245 2024-11-19T21:41:13,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f921280b1c3745069db91df1d359a245, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T21:41:13,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/acb50c7007344ff3b738211eee8fa73f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/acb50c7007344ff3b738211eee8fa73f 2024-11-19T21:41:13,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/acb50c7007344ff3b738211eee8fa73f, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T21:41:13,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d5c69b89f3a83a34d8bee2d0fed83b40 in 67ms, sequenceid=287, compaction requested=true 2024-11-19T21:41:13,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:13,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:13,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:13,269 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:13,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:13,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:13,269 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:13,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:13,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:13,270 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:13,270 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:13,270 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:13,270 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:13,270 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:13,270 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:13,270 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f198f009a7554fb4ab0291395628c164, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/99931e6f1bf84b49a88d94f4dc195f0b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d190636a575d4e57ae28bf895b9ec333] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.5 K 2024-11-19T21:41:13,270 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/01b91f291e1849a3aea33c9ac156fe4c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2e29aed923874702a1ffe4fa3bdb0ee7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f921280b1c3745069db91df1d359a245] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.5 K 2024-11-19T21:41:13,270 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 01b91f291e1849a3aea33c9ac156fe4c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732052469024 2024-11-19T21:41:13,270 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting f198f009a7554fb4ab0291395628c164, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732052469024 2024-11-19T21:41:13,271 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e29aed923874702a1ffe4fa3bdb0ee7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732052470582 2024-11-19T21:41:13,271 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99931e6f1bf84b49a88d94f4dc195f0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732052470582 2024-11-19T21:41:13,271 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting f921280b1c3745069db91df1d359a245, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732052472778 2024-11-19T21:41:13,271 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting d190636a575d4e57ae28bf895b9ec333, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732052472778 2024-11-19T21:41:13,284 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:13,285 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/f62d125d1c6140af98b024e71bf25b9c is 50, key is test_row_0/B:col10/1732052473201/Put/seqid=0 2024-11-19T21:41:13,287 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:13,288 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/9dfd73c8a57d48eb9f7ef9edece250ea is 50, key is test_row_0/A:col10/1732052473201/Put/seqid=0 2024-11-19T21:41:13,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742147_1323 (size=12983) 2024-11-19T21:41:13,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742148_1324 (size=12983) 2024-11-19T21:41:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:13,361 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:13,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T21:41:13,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/21259ac11aa2472cb9f6f3e3ac619036 is 50, key is test_row_0/A:col10/1732052473244/Put/seqid=0 2024-11-19T21:41:13,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742149_1325 (size=14741) 2024-11-19T21:41:13,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052533388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052533393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052533408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052533408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052533498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052533508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,697 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/f62d125d1c6140af98b024e71bf25b9c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f62d125d1c6140af98b024e71bf25b9c 2024-11-19T21:41:13,701 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into f62d125d1c6140af98b024e71bf25b9c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:13,701 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:13,701 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=13, startTime=1732052473269; duration=0sec 2024-11-19T21:41:13,701 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:13,701 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:13,701 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:13,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:13,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:13,703 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:13,703 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/398ab7fef2b94d39adbe952b40b85a15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8ce40b590ab84bf79185acbae79cedee, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/acb50c7007344ff3b738211eee8fa73f] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.5 K 2024-11-19T21:41:13,704 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 398ab7fef2b94d39adbe952b40b85a15, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732052469024 2024-11-19T21:41:13,704 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce40b590ab84bf79185acbae79cedee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732052470582 2024-11-19T21:41:13,704 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting acb50c7007344ff3b738211eee8fa73f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732052472778 2024-11-19T21:41:13,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052533708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,714 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/9dfd73c8a57d48eb9f7ef9edece250ea as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/9dfd73c8a57d48eb9f7ef9edece250ea 2024-11-19T21:41:13,721 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into 9dfd73c8a57d48eb9f7ef9edece250ea(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:13,721 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:13,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,721 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=13, startTime=1732052473269; duration=0sec 2024-11-19T21:41:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052533718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,721 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:13,721 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,729 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:13,730 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/e59c786d9b564beab339d931f30c8f03 is 50, key is test_row_0/C:col10/1732052473201/Put/seqid=0 2024-11-19T21:41:13,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742150_1326 (size=12983) 2024-11-19T21:41:13,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/21259ac11aa2472cb9f6f3e3ac619036 2024-11-19T21:41:13,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7af69dffe58f42eabab522b7b92d1de1 is 50, key is test_row_0/B:col10/1732052473244/Put/seqid=0 2024-11-19T21:41:13,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742151_1327 (size=12301) 2024-11-19T21:41:13,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7af69dffe58f42eabab522b7b92d1de1 2024-11-19T21:41:13,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/50c86bbb504148eab690078d2fd39d98 is 50, key is test_row_0/C:col10/1732052473244/Put/seqid=0 2024-11-19T21:41:13,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742152_1328 (size=12301) 2024-11-19T21:41:13,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052533918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052533918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052534018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052534028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,158 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/e59c786d9b564beab339d931f30c8f03 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/e59c786d9b564beab339d931f30c8f03 2024-11-19T21:41:14,163 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into e59c786d9b564beab339d931f30c8f03(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:14,163 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:14,163 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=13, startTime=1732052473269; duration=0sec 2024-11-19T21:41:14,163 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:14,163 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:14,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/50c86bbb504148eab690078d2fd39d98 2024-11-19T21:41:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/21259ac11aa2472cb9f6f3e3ac619036 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/21259ac11aa2472cb9f6f3e3ac619036 2024-11-19T21:41:14,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/21259ac11aa2472cb9f6f3e3ac619036, entries=200, sequenceid=309, filesize=14.4 K 2024-11-19T21:41:14,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7af69dffe58f42eabab522b7b92d1de1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7af69dffe58f42eabab522b7b92d1de1 2024-11-19T21:41:14,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7af69dffe58f42eabab522b7b92d1de1, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T21:41:14,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/50c86bbb504148eab690078d2fd39d98 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/50c86bbb504148eab690078d2fd39d98 2024-11-19T21:41:14,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/50c86bbb504148eab690078d2fd39d98, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T21:41:14,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d5c69b89f3a83a34d8bee2d0fed83b40 in 870ms, sequenceid=309, compaction requested=false 2024-11-19T21:41:14,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:14,530 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:14,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T21:41:14,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/16fff2ec3df7404aa26ac1cc429548bb is 50, key is test_row_0/A:col10/1732052473386/Put/seqid=0 2024-11-19T21:41:14,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742153_1329 (size=14741) 2024-11-19T21:41:14,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/16fff2ec3df7404aa26ac1cc429548bb 2024-11-19T21:41:14,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/4a90302483dc43d3aca850965f23622f is 50, key is test_row_0/B:col10/1732052473386/Put/seqid=0 2024-11-19T21:41:14,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742154_1330 (size=12301) 2024-11-19T21:41:14,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/4a90302483dc43d3aca850965f23622f 2024-11-19T21:41:14,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0cb0358a69a14d92906c83757f46eea8 is 50, key is test_row_0/C:col10/1732052473386/Put/seqid=0 2024-11-19T21:41:14,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742155_1331 (size=12301) 2024-11-19T21:41:14,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0cb0358a69a14d92906c83757f46eea8 2024-11-19T21:41:14,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/16fff2ec3df7404aa26ac1cc429548bb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/16fff2ec3df7404aa26ac1cc429548bb 2024-11-19T21:41:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052534577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052534581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/16fff2ec3df7404aa26ac1cc429548bb, entries=200, sequenceid=327, filesize=14.4 K 2024-11-19T21:41:14,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/4a90302483dc43d3aca850965f23622f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4a90302483dc43d3aca850965f23622f 2024-11-19T21:41:14,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4a90302483dc43d3aca850965f23622f, entries=150, sequenceid=327, filesize=12.0 K 2024-11-19T21:41:14,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0cb0358a69a14d92906c83757f46eea8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0cb0358a69a14d92906c83757f46eea8 2024-11-19T21:41:14,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0cb0358a69a14d92906c83757f46eea8, entries=150, sequenceid=327, filesize=12.0 K 2024-11-19T21:41:14,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d5c69b89f3a83a34d8bee2d0fed83b40 in 72ms, sequenceid=327, compaction requested=true 2024-11-19T21:41:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:14,603 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:14,603 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:14,604 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:14,604 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:14,604 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:14,604 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f62d125d1c6140af98b024e71bf25b9c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7af69dffe58f42eabab522b7b92d1de1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4a90302483dc43d3aca850965f23622f] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.7 K 2024-11-19T21:41:14,604 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:14,604 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:14,604 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:14,605 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/9dfd73c8a57d48eb9f7ef9edece250ea, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/21259ac11aa2472cb9f6f3e3ac619036, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/16fff2ec3df7404aa26ac1cc429548bb] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=41.5 K 2024-11-19T21:41:14,605 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting f62d125d1c6140af98b024e71bf25b9c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732052472778 2024-11-19T21:41:14,605 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dfd73c8a57d48eb9f7ef9edece250ea, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732052472778 2024-11-19T21:41:14,605 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af69dffe58f42eabab522b7b92d1de1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732052473227 2024-11-19T21:41:14,605 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21259ac11aa2472cb9f6f3e3ac619036, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732052473227 2024-11-19T21:41:14,605 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a90302483dc43d3aca850965f23622f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732052473381 2024-11-19T21:41:14,605 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16fff2ec3df7404aa26ac1cc429548bb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732052473381 2024-11-19T21:41:14,624 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:14,625 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:14,625 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/22174094fccd48a5aab489af10d0c567 is 50, key is test_row_0/B:col10/1732052473386/Put/seqid=0 2024-11-19T21:41:14,625 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/7149a90dd9ad48429b2b011e31f450be is 50, key is test_row_0/A:col10/1732052473386/Put/seqid=0 2024-11-19T21:41:14,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742156_1332 (size=13085) 2024-11-19T21:41:14,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742157_1333 (size=13085) 2024-11-19T21:41:14,642 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/7149a90dd9ad48429b2b011e31f450be as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7149a90dd9ad48429b2b011e31f450be 2024-11-19T21:41:14,650 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into 7149a90dd9ad48429b2b011e31f450be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:14,650 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:14,650 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=13, startTime=1732052474603; duration=0sec 2024-11-19T21:41:14,650 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:14,650 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:14,650 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:14,652 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:14,652 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:14,652 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:14,652 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/e59c786d9b564beab339d931f30c8f03, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/50c86bbb504148eab690078d2fd39d98, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0cb0358a69a14d92906c83757f46eea8] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.7 K 2024-11-19T21:41:14,653 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting e59c786d9b564beab339d931f30c8f03, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732052472778 2024-11-19T21:41:14,653 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c86bbb504148eab690078d2fd39d98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732052473227 2024-11-19T21:41:14,653 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cb0358a69a14d92906c83757f46eea8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732052473381 2024-11-19T21:41:14,659 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:14,660 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/d030a9b89b184a9fa64cbeda1c038353 is 50, key is test_row_0/C:col10/1732052473386/Put/seqid=0 2024-11-19T21:41:14,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742158_1334 (size=13085) 2024-11-19T21:41:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:14,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T21:41:14,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/2bc6b95b14604d58b542c847cbd37e59 is 50, key is test_row_0/A:col10/1732052474571/Put/seqid=0 2024-11-19T21:41:14,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742159_1335 (size=12301) 2024-11-19T21:41:14,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/2bc6b95b14604d58b542c847cbd37e59 2024-11-19T21:41:14,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/c303397e984d434e963c569852ffdf46 is 50, key is test_row_0/B:col10/1732052474571/Put/seqid=0 2024-11-19T21:41:14,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742160_1336 (size=12301) 2024-11-19T21:41:14,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/c303397e984d434e963c569852ffdf46 2024-11-19T21:41:14,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/c82eadb1877a4d4583e4e256f9444ebb is 50, key is test_row_0/C:col10/1732052474571/Put/seqid=0 2024-11-19T21:41:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052534752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052534753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742161_1337 (size=12301) 2024-11-19T21:41:14,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/c82eadb1877a4d4583e4e256f9444ebb 2024-11-19T21:41:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/2bc6b95b14604d58b542c847cbd37e59 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/2bc6b95b14604d58b542c847cbd37e59 2024-11-19T21:41:14,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/2bc6b95b14604d58b542c847cbd37e59, entries=150, sequenceid=350, filesize=12.0 K 2024-11-19T21:41:14,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/c303397e984d434e963c569852ffdf46 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/c303397e984d434e963c569852ffdf46 2024-11-19T21:41:14,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/c303397e984d434e963c569852ffdf46, entries=150, sequenceid=350, filesize=12.0 K 2024-11-19T21:41:14,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/c82eadb1877a4d4583e4e256f9444ebb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c82eadb1877a4d4583e4e256f9444ebb 2024-11-19T21:41:14,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c82eadb1877a4d4583e4e256f9444ebb, entries=150, sequenceid=350, filesize=12.0 K 2024-11-19T21:41:14,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for d5c69b89f3a83a34d8bee2d0fed83b40 in 105ms, sequenceid=350, compaction requested=false 2024-11-19T21:41:14,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T21:41:14,848 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:14,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T21:41:14,850 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T21:41:14,850 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:14,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:14,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T21:41:14,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/12f6c56caf8d4e83873637087e520470 is 50, key is test_row_0/A:col10/1732052474860/Put/seqid=0 2024-11-19T21:41:14,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742162_1338 (size=14741) 2024-11-19T21:41:14,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/12f6c56caf8d4e83873637087e520470 2024-11-19T21:41:14,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/13594de9bc8a4865b90a74fd713b5ce3 is 50, key is test_row_0/B:col10/1732052474860/Put/seqid=0 2024-11-19T21:41:14,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052534934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052534935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052534939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052534939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:14,945 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:14,945 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T21:41:14,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742163_1339 (size=12301) 2024-11-19T21:41:15,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T21:41:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,034 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/22174094fccd48a5aab489af10d0c567 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/22174094fccd48a5aab489af10d0c567 2024-11-19T21:41:15,041 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into 22174094fccd48a5aab489af10d0c567(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:15,041 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:15,041 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=13, startTime=1732052474603; duration=0sec 2024-11-19T21:41:15,042 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:15,042 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052535049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:15,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052535049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:15,071 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/d030a9b89b184a9fa64cbeda1c038353 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d030a9b89b184a9fa64cbeda1c038353 2024-11-19T21:41:15,076 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into d030a9b89b184a9fa64cbeda1c038353(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:15,076 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:15,076 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=13, startTime=1732052474603; duration=0sec 2024-11-19T21:41:15,076 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:15,076 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:15,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T21:41:15,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:15,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T21:41:15,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052535258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:15,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:15,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052535258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:15,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:15,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T21:41:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/13594de9bc8a4865b90a74fd713b5ce3 2024-11-19T21:41:15,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0060a4b42ed24fa3bec38322e5bad795 is 50, key is test_row_0/C:col10/1732052474860/Put/seqid=0 2024-11-19T21:41:15,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742164_1340 (size=12301) 2024-11-19T21:41:15,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T21:41:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T21:41:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052535568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052535568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:15,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T21:41:15,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:15,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T21:41:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:15,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0060a4b42ed24fa3bec38322e5bad795 2024-11-19T21:41:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/12f6c56caf8d4e83873637087e520470 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/12f6c56caf8d4e83873637087e520470 2024-11-19T21:41:15,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/12f6c56caf8d4e83873637087e520470, entries=200, sequenceid=366, filesize=14.4 K 2024-11-19T21:41:15,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/13594de9bc8a4865b90a74fd713b5ce3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/13594de9bc8a4865b90a74fd713b5ce3 2024-11-19T21:41:15,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/13594de9bc8a4865b90a74fd713b5ce3, entries=150, sequenceid=366, filesize=12.0 K 2024-11-19T21:41:15,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/0060a4b42ed24fa3bec38322e5bad795 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0060a4b42ed24fa3bec38322e5bad795 2024-11-19T21:41:15,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0060a4b42ed24fa3bec38322e5bad795, entries=150, sequenceid=366, filesize=12.0 K 2024-11-19T21:41:15,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d5c69b89f3a83a34d8bee2d0fed83b40 in 974ms, sequenceid=366, compaction requested=true 2024-11-19T21:41:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:15,836 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:15,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:15,837 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:15,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:15,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:15,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:15,838 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:15,838 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:15,838 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:15,838 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:15,838 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,838 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/22174094fccd48a5aab489af10d0c567, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/c303397e984d434e963c569852ffdf46, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/13594de9bc8a4865b90a74fd713b5ce3] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.8 K 2024-11-19T21:41:15,838 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,838 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7149a90dd9ad48429b2b011e31f450be, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/2bc6b95b14604d58b542c847cbd37e59, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/12f6c56caf8d4e83873637087e520470] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=39.2 K 2024-11-19T21:41:15,839 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 22174094fccd48a5aab489af10d0c567, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732052473381 2024-11-19T21:41:15,839 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7149a90dd9ad48429b2b011e31f450be, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732052473381 2024-11-19T21:41:15,839 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bc6b95b14604d58b542c847cbd37e59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732052474567 2024-11-19T21:41:15,839 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c303397e984d434e963c569852ffdf46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732052474567 2024-11-19T21:41:15,842 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f6c56caf8d4e83873637087e520470, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732052474706 2024-11-19T21:41:15,842 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 13594de9bc8a4865b90a74fd713b5ce3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732052474717 2024-11-19T21:41:15,860 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:15,861 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/efe0b6e95b6047e9bcffb354c1ce4aa4 is 50, key is test_row_0/B:col10/1732052474860/Put/seqid=0 2024-11-19T21:41:15,873 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:15,873 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/7297424da29f40ed8ffb1ecd5b47d4fd is 50, key is test_row_0/A:col10/1732052474860/Put/seqid=0 2024-11-19T21:41:15,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742165_1341 (size=13187) 2024-11-19T21:41:15,897 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/efe0b6e95b6047e9bcffb354c1ce4aa4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/efe0b6e95b6047e9bcffb354c1ce4aa4 2024-11-19T21:41:15,907 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into efe0b6e95b6047e9bcffb354c1ce4aa4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:15,907 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:15,907 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=13, startTime=1732052475836; duration=0sec 2024-11-19T21:41:15,907 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:15,907 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:15,907 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:15,908 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:15,908 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:15,908 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,909 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d030a9b89b184a9fa64cbeda1c038353, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c82eadb1877a4d4583e4e256f9444ebb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0060a4b42ed24fa3bec38322e5bad795] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.8 K 2024-11-19T21:41:15,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742166_1342 (size=13187) 2024-11-19T21:41:15,909 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d030a9b89b184a9fa64cbeda1c038353, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732052473381 2024-11-19T21:41:15,911 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c82eadb1877a4d4583e4e256f9444ebb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732052474567 2024-11-19T21:41:15,911 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0060a4b42ed24fa3bec38322e5bad795, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732052474717 2024-11-19T21:41:15,923 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/7297424da29f40ed8ffb1ecd5b47d4fd as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7297424da29f40ed8ffb1ecd5b47d4fd 2024-11-19T21:41:15,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T21:41:15,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:15,927 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T21:41:15,929 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into 7297424da29f40ed8ffb1ecd5b47d4fd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:15,929 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:15,929 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=13, startTime=1732052475836; duration=0sec 2024-11-19T21:41:15,929 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:15,929 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:15,942 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:15,943 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/853233576f6e4904b0d79f0b4f60bc74 is 50, key is test_row_0/C:col10/1732052474860/Put/seqid=0 2024-11-19T21:41:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f54530beab9b4a798a62b77421228d0c is 50, key is test_row_0/A:col10/1732052474931/Put/seqid=0 2024-11-19T21:41:15,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742167_1343 (size=12301) 2024-11-19T21:41:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T21:41:15,990 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f54530beab9b4a798a62b77421228d0c 2024-11-19T21:41:16,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742168_1344 (size=13187) 2024-11-19T21:41:16,009 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/853233576f6e4904b0d79f0b4f60bc74 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/853233576f6e4904b0d79f0b4f60bc74 2024-11-19T21:41:16,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/e18f958d5ed94c7e8f3858762c091d79 is 50, key is test_row_0/B:col10/1732052474931/Put/seqid=0 2024-11-19T21:41:16,017 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into 853233576f6e4904b0d79f0b4f60bc74(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:16,018 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:16,018 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=13, startTime=1732052475837; duration=0sec 2024-11-19T21:41:16,018 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:16,018 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:16,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742169_1345 (size=12301) 2024-11-19T21:41:16,062 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/e18f958d5ed94c7e8f3858762c091d79 2024-11-19T21:41:16,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/dca2a05fc34247798036b20e9829d146 is 50, key is test_row_0/C:col10/1732052474931/Put/seqid=0 2024-11-19T21:41:16,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:16,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742170_1346 (size=12301) 2024-11-19T21:41:16,108 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/dca2a05fc34247798036b20e9829d146 2024-11-19T21:41:16,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f54530beab9b4a798a62b77421228d0c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f54530beab9b4a798a62b77421228d0c 2024-11-19T21:41:16,117 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f54530beab9b4a798a62b77421228d0c, entries=150, sequenceid=389, filesize=12.0 K 2024-11-19T21:41:16,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/e18f958d5ed94c7e8f3858762c091d79 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e18f958d5ed94c7e8f3858762c091d79 2024-11-19T21:41:16,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052536114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052536115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,124 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e18f958d5ed94c7e8f3858762c091d79, entries=150, sequenceid=389, filesize=12.0 K 2024-11-19T21:41:16,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/dca2a05fc34247798036b20e9829d146 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/dca2a05fc34247798036b20e9829d146 2024-11-19T21:41:16,129 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/dca2a05fc34247798036b20e9829d146, entries=150, sequenceid=389, filesize=12.0 K 2024-11-19T21:41:16,130 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for d5c69b89f3a83a34d8bee2d0fed83b40 in 203ms, sequenceid=389, compaction requested=false 2024-11-19T21:41:16,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:16,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:16,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T21:41:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T21:41:16,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T21:41:16,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2810 sec 2024-11-19T21:41:16,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.2860 sec 2024-11-19T21:41:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:16,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:16,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T21:41:16,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/c22c155083a64269baba22618214e9e1 is 50, key is test_row_0/A:col10/1732052476230/Put/seqid=0 2024-11-19T21:41:16,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742171_1347 (size=17181) 2024-11-19T21:41:16,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052536294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052536294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052536409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052536409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052536618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052536618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/c22c155083a64269baba22618214e9e1 2024-11-19T21:41:16,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/10aeff4796fd4c65a91f08285f42a351 is 50, key is test_row_0/B:col10/1732052476230/Put/seqid=0 2024-11-19T21:41:16,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742172_1348 (size=12301) 2024-11-19T21:41:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052536928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052536928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052536948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,952 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052536968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:16,974 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:16,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:16,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T21:41:16,998 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:16,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-19T21:41:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T21:41:17,002 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:17,003 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:17,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:17,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/10aeff4796fd4c65a91f08285f42a351 2024-11-19T21:41:17,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/7e9d90013b61406fa8b5d126f425bc16 is 50, key is test_row_0/C:col10/1732052476230/Put/seqid=0 2024-11-19T21:41:17,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742173_1349 (size=12301) 2024-11-19T21:41:17,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/7e9d90013b61406fa8b5d126f425bc16 2024-11-19T21:41:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T21:41:17,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/c22c155083a64269baba22618214e9e1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c22c155083a64269baba22618214e9e1 2024-11-19T21:41:17,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c22c155083a64269baba22618214e9e1, entries=250, sequenceid=410, filesize=16.8 K 2024-11-19T21:41:17,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/10aeff4796fd4c65a91f08285f42a351 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/10aeff4796fd4c65a91f08285f42a351 2024-11-19T21:41:17,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/10aeff4796fd4c65a91f08285f42a351, entries=150, sequenceid=410, filesize=12.0 K 2024-11-19T21:41:17,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/7e9d90013b61406fa8b5d126f425bc16 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e9d90013b61406fa8b5d126f425bc16 2024-11-19T21:41:17,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e9d90013b61406fa8b5d126f425bc16, entries=150, sequenceid=410, filesize=12.0 K 2024-11-19T21:41:17,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d5c69b89f3a83a34d8bee2d0fed83b40 in 912ms, sequenceid=410, compaction requested=true 2024-11-19T21:41:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:17,144 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T21:41:17,144 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:17,147 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:17,147 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:17,147 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:17,148 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7297424da29f40ed8ffb1ecd5b47d4fd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f54530beab9b4a798a62b77421228d0c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c22c155083a64269baba22618214e9e1] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=41.7 K 2024-11-19T21:41:17,148 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:17,148 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:17,148 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:17,148 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/efe0b6e95b6047e9bcffb354c1ce4aa4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e18f958d5ed94c7e8f3858762c091d79, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/10aeff4796fd4c65a91f08285f42a351] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.9 K 2024-11-19T21:41:17,148 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting efe0b6e95b6047e9bcffb354c1ce4aa4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732052474717 2024-11-19T21:41:17,149 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7297424da29f40ed8ffb1ecd5b47d4fd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732052474717 2024-11-19T21:41:17,149 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e18f958d5ed94c7e8f3858762c091d79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732052474924 2024-11-19T21:41:17,149 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 10aeff4796fd4c65a91f08285f42a351, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732052476112 2024-11-19T21:41:17,149 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting f54530beab9b4a798a62b77421228d0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732052474924 2024-11-19T21:41:17,150 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22c155083a64269baba22618214e9e1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732052476106 2024-11-19T21:41:17,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T21:41:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:17,157 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T21:41:17,174 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:17,175 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/430028ee05c74086b4c5d9242de217d7 is 50, key is test_row_0/B:col10/1732052476230/Put/seqid=0 2024-11-19T21:41:17,181 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:17,181 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/8069287b9f664f819d63ea61fc1454d6 is 50, key is test_row_0/A:col10/1732052476230/Put/seqid=0 2024-11-19T21:41:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f06c30d061d340d1acafecf1538a8007 is 50, key is test_row_0/A:col10/1732052476286/Put/seqid=0 2024-11-19T21:41:17,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742174_1350 (size=13289) 2024-11-19T21:41:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:17,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:17,226 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/430028ee05c74086b4c5d9242de217d7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/430028ee05c74086b4c5d9242de217d7 2024-11-19T21:41:17,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742175_1351 (size=13289) 2024-11-19T21:41:17,235 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into 430028ee05c74086b4c5d9242de217d7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:17,235 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:17,235 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=13, startTime=1732052477144; duration=0sec 2024-11-19T21:41:17,236 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:17,236 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:17,236 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:17,238 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:17,238 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:17,238 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:17,238 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/853233576f6e4904b0d79f0b4f60bc74, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/dca2a05fc34247798036b20e9829d146, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e9d90013b61406fa8b5d126f425bc16] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=36.9 K 2024-11-19T21:41:17,239 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 853233576f6e4904b0d79f0b4f60bc74, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732052474717 2024-11-19T21:41:17,239 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting dca2a05fc34247798036b20e9829d146, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732052474924 2024-11-19T21:41:17,241 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/8069287b9f664f819d63ea61fc1454d6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/8069287b9f664f819d63ea61fc1454d6 2024-11-19T21:41:17,243 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e9d90013b61406fa8b5d126f425bc16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732052476112 2024-11-19T21:41:17,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742176_1352 (size=12301) 2024-11-19T21:41:17,247 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f06c30d061d340d1acafecf1538a8007 2024-11-19T21:41:17,252 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into 8069287b9f664f819d63ea61fc1454d6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:17,252 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:17,252 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=13, startTime=1732052477144; duration=0sec 2024-11-19T21:41:17,252 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:17,252 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:17,265 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:17,266 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/80b2463607374d1aa7d444cc371ad879 is 50, key is test_row_0/C:col10/1732052476230/Put/seqid=0 2024-11-19T21:41:17,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/ec5b8b5d92854cf4b2080ebe985e7f18 is 50, key is test_row_0/B:col10/1732052476286/Put/seqid=0 2024-11-19T21:41:17,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742177_1353 (size=12301) 2024-11-19T21:41:17,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052537306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:17,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742178_1354 (size=13289) 2024-11-19T21:41:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T21:41:17,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052537418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:17,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052537438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:17,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052537438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T21:41:17,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052537628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:17,698 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/ec5b8b5d92854cf4b2080ebe985e7f18 2024-11-19T21:41:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/03051d0db2494e0d8a7b1fba3f3c0cec is 50, key is test_row_0/C:col10/1732052476286/Put/seqid=0 2024-11-19T21:41:17,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742179_1355 (size=12301) 2024-11-19T21:41:17,731 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/80b2463607374d1aa7d444cc371ad879 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/80b2463607374d1aa7d444cc371ad879 2024-11-19T21:41:17,735 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into 80b2463607374d1aa7d444cc371ad879(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:17,735 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:17,735 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=13, startTime=1732052477144; duration=0sec 2024-11-19T21:41:17,735 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:17,736 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:17,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052537939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,109 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/03051d0db2494e0d8a7b1fba3f3c0cec 2024-11-19T21:41:18,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f06c30d061d340d1acafecf1538a8007 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f06c30d061d340d1acafecf1538a8007 2024-11-19T21:41:18,137 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f06c30d061d340d1acafecf1538a8007, entries=150, sequenceid=428, filesize=12.0 K 2024-11-19T21:41:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T21:41:18,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/ec5b8b5d92854cf4b2080ebe985e7f18 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ec5b8b5d92854cf4b2080ebe985e7f18 2024-11-19T21:41:18,144 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ec5b8b5d92854cf4b2080ebe985e7f18, entries=150, sequenceid=428, filesize=12.0 K 2024-11-19T21:41:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/03051d0db2494e0d8a7b1fba3f3c0cec as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/03051d0db2494e0d8a7b1fba3f3c0cec 2024-11-19T21:41:18,150 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/03051d0db2494e0d8a7b1fba3f3c0cec, entries=150, sequenceid=428, filesize=12.0 K 2024-11-19T21:41:18,151 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d5c69b89f3a83a34d8bee2d0fed83b40 in 994ms, sequenceid=428, compaction requested=false 2024-11-19T21:41:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-19T21:41:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-19T21:41:18,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T21:41:18,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1500 sec 2024-11-19T21:41:18,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.1570 sec 2024-11-19T21:41:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:18,450 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:18,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T21:41:18,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/54620dc4c69149d0801e450c9d4e0a40 is 50, key is test_row_0/A:col10/1732052478450/Put/seqid=0 2024-11-19T21:41:18,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742180_1356 (size=17181) 2024-11-19T21:41:18,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052538467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:18,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052538468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052538469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052538578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052538578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052538578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052538788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:18,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052538788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052538788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:18,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:18,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/54620dc4c69149d0801e450c9d4e0a40 2024-11-19T21:41:18,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/56cc94aca8ef4934b6a087d3b064e0eb is 50, key is test_row_0/B:col10/1732052478450/Put/seqid=0 2024-11-19T21:41:18,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742181_1357 (size=12301) 2024-11-19T21:41:19,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052539098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052539098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052539098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T21:41:19,158 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:19,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-19T21:41:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T21:41:19,162 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:19,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:19,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T21:41:19,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/56cc94aca8ef4934b6a087d3b064e0eb 2024-11-19T21:41:19,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/ad06c7f2511e4b2aae4176b3e81673bd is 50, key is test_row_0/C:col10/1732052478450/Put/seqid=0 2024-11-19T21:41:19,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742182_1358 (size=12301) 2024-11-19T21:41:19,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/ad06c7f2511e4b2aae4176b3e81673bd 2024-11-19T21:41:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/54620dc4c69149d0801e450c9d4e0a40 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/54620dc4c69149d0801e450c9d4e0a40 2024-11-19T21:41:19,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/54620dc4c69149d0801e450c9d4e0a40, entries=250, sequenceid=452, filesize=16.8 K 2024-11-19T21:41:19,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/56cc94aca8ef4934b6a087d3b064e0eb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/56cc94aca8ef4934b6a087d3b064e0eb 2024-11-19T21:41:19,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T21:41:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:19,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:19,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:19,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:19,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/56cc94aca8ef4934b6a087d3b064e0eb, entries=150, sequenceid=452, filesize=12.0 K 2024-11-19T21:41:19,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/ad06c7f2511e4b2aae4176b3e81673bd as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/ad06c7f2511e4b2aae4176b3e81673bd 2024-11-19T21:41:19,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/ad06c7f2511e4b2aae4176b3e81673bd, entries=150, sequenceid=452, filesize=12.0 K 2024-11-19T21:41:19,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d5c69b89f3a83a34d8bee2d0fed83b40 in 883ms, sequenceid=452, compaction requested=true 2024-11-19T21:41:19,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:19,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:19,333 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:19,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:19,333 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:19,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:19,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:19,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:19,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:19,334 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:19,334 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:19,334 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:19,335 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/8069287b9f664f819d63ea61fc1454d6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f06c30d061d340d1acafecf1538a8007, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/54620dc4c69149d0801e450c9d4e0a40] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=41.8 K 2024-11-19T21:41:19,335 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:19,336 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:19,336 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:19,336 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/430028ee05c74086b4c5d9242de217d7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ec5b8b5d92854cf4b2080ebe985e7f18, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/56cc94aca8ef4934b6a087d3b064e0eb] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=37.0 K 2024-11-19T21:41:19,336 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8069287b9f664f819d63ea61fc1454d6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732052476112 2024-11-19T21:41:19,336 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 430028ee05c74086b4c5d9242de217d7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732052476112 2024-11-19T21:41:19,336 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting f06c30d061d340d1acafecf1538a8007, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732052476286 2024-11-19T21:41:19,336 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54620dc4c69149d0801e450c9d4e0a40, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732052477285 2024-11-19T21:41:19,336 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5b8b5d92854cf4b2080ebe985e7f18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732052476286 2024-11-19T21:41:19,337 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 56cc94aca8ef4934b6a087d3b064e0eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732052477299 2024-11-19T21:41:19,344 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:19,345 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/2d2d0a3a475f42168830c81c322aad40 is 50, key is test_row_0/B:col10/1732052478450/Put/seqid=0 2024-11-19T21:41:19,347 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:19,348 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/dc71d0511f8f442db72e81fbc6d3d45b is 50, key is test_row_0/A:col10/1732052478450/Put/seqid=0 2024-11-19T21:41:19,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742183_1359 (size=13391) 2024-11-19T21:41:19,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742184_1360 (size=13391) 2024-11-19T21:41:19,383 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/dc71d0511f8f442db72e81fbc6d3d45b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/dc71d0511f8f442db72e81fbc6d3d45b 2024-11-19T21:41:19,388 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into dc71d0511f8f442db72e81fbc6d3d45b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:19,388 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:19,388 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=13, startTime=1732052479333; duration=0sec 2024-11-19T21:41:19,389 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:19,389 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:19,389 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:19,390 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:19,390 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:19,390 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:19,391 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/80b2463607374d1aa7d444cc371ad879, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/03051d0db2494e0d8a7b1fba3f3c0cec, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/ad06c7f2511e4b2aae4176b3e81673bd] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=37.0 K 2024-11-19T21:41:19,391 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b2463607374d1aa7d444cc371ad879, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732052476112 2024-11-19T21:41:19,391 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03051d0db2494e0d8a7b1fba3f3c0cec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732052476286 2024-11-19T21:41:19,392 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad06c7f2511e4b2aae4176b3e81673bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732052477299 2024-11-19T21:41:19,398 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:19,399 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/d131b0f902dc4b5b9adf8d8c363d1917 is 50, key is test_row_0/C:col10/1732052478450/Put/seqid=0 2024-11-19T21:41:19,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742185_1361 (size=13391) 2024-11-19T21:41:19,424 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/d131b0f902dc4b5b9adf8d8c363d1917 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d131b0f902dc4b5b9adf8d8c363d1917 2024-11-19T21:41:19,429 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into d131b0f902dc4b5b9adf8d8c363d1917(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:19,429 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:19,429 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=13, startTime=1732052479333; duration=0sec 2024-11-19T21:41:19,429 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:19,429 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:19,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T21:41:19,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:19,477 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T21:41:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T21:41:19,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/05894cfdc56841be87be4005f1fb86d0 is 50, key is test_row_0/A:col10/1732052478462/Put/seqid=0 2024-11-19T21:41:19,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742186_1362 (size=12301) 2024-11-19T21:41:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:19,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. as already flushing 2024-11-19T21:41:19,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052539648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052539650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052539651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052539758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052539758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052539758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,769 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/2d2d0a3a475f42168830c81c322aad40 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2d2d0a3a475f42168830c81c322aad40 2024-11-19T21:41:19,773 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into 2d2d0a3a475f42168830c81c322aad40(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:19,773 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:19,773 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=13, startTime=1732052479333; duration=0sec 2024-11-19T21:41:19,773 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:19,773 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T21:41:19,932 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/05894cfdc56841be87be4005f1fb86d0 2024-11-19T21:41:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/4c94d8b432bc4282957d1aafd60c8142 is 50, key is test_row_0/B:col10/1732052478462/Put/seqid=0 2024-11-19T21:41:19,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052539969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052539968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052539969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:19,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742187_1363 (size=12301) 2024-11-19T21:41:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052540278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052540278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052540278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T21:41:20,386 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/4c94d8b432bc4282957d1aafd60c8142 2024-11-19T21:41:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/c1eae4a3716841dc855e0e7ed7feecfe is 50, key is test_row_0/C:col10/1732052478462/Put/seqid=0 2024-11-19T21:41:20,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742188_1364 (size=12301) 2024-11-19T21:41:20,400 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/c1eae4a3716841dc855e0e7ed7feecfe 2024-11-19T21:41:20,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/05894cfdc56841be87be4005f1fb86d0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/05894cfdc56841be87be4005f1fb86d0 2024-11-19T21:41:20,410 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/05894cfdc56841be87be4005f1fb86d0, entries=150, sequenceid=469, filesize=12.0 K 2024-11-19T21:41:20,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/4c94d8b432bc4282957d1aafd60c8142 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4c94d8b432bc4282957d1aafd60c8142 2024-11-19T21:41:20,414 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4c94d8b432bc4282957d1aafd60c8142, entries=150, sequenceid=469, filesize=12.0 K 2024-11-19T21:41:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/c1eae4a3716841dc855e0e7ed7feecfe as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c1eae4a3716841dc855e0e7ed7feecfe 2024-11-19T21:41:20,420 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c1eae4a3716841dc855e0e7ed7feecfe, entries=150, sequenceid=469, filesize=12.0 K 2024-11-19T21:41:20,420 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d5c69b89f3a83a34d8bee2d0fed83b40 in 943ms, sequenceid=469, compaction requested=false 2024-11-19T21:41:20,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:20,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:20,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-19T21:41:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-19T21:41:20,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T21:41:20,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2590 sec 2024-11-19T21:41:20,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.2650 sec 2024-11-19T21:41:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:20,789 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:20,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T21:41:20,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f33a4985f24f46f6a234ee271a2b1e0b is 50, key is test_row_0/A:col10/1732052480789/Put/seqid=0 2024-11-19T21:41:20,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052540800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052540803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052540806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742189_1365 (size=14741) 2024-11-19T21:41:20,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f33a4985f24f46f6a234ee271a2b1e0b 2024-11-19T21:41:20,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7feb02de4533455b96b1695436776e56 is 50, key is test_row_0/B:col10/1732052480789/Put/seqid=0 2024-11-19T21:41:20,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742190_1366 (size=12301) 2024-11-19T21:41:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052540908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052540918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052540918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,952 INFO [Thread-1217 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1217. 2024-11-19T21:41:20,952 DEBUG [Thread-1217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:20,952 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,952 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:20,954 INFO [Thread-1219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1219. 2024-11-19T21:41:20,954 DEBUG [Thread-1219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:20,955 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,955 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:20,955 INFO [Thread-1223 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1223. 2024-11-19T21:41:20,955 DEBUG [Thread-1223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:20,955 DEBUG [Thread-1223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,955 DEBUG [Thread-1223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:20,955 INFO [Thread-1225 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1225. 2024-11-19T21:41:20,955 DEBUG [Thread-1225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:20,955 DEBUG [Thread-1225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,955 DEBUG [Thread-1225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:20,958 INFO [Thread-1221 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1221. 2024-11-19T21:41:20,958 DEBUG [Thread-1221 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:20,958 DEBUG [Thread-1221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,958 DEBUG [Thread-1221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:20,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48026 deadline: 1732052540969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,969 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:20,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48068 deadline: 1732052540988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:20,989 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in d5c69b89f3a83a34d8bee2d0fed83b40 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:20,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:20,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:21,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052541118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:21,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:21,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052541128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052541128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:21,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7feb02de4533455b96b1695436776e56 2024-11-19T21:41:21,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/fe4cce5290b2411ab69ec8463225dceb is 50, key is test_row_0/C:col10/1732052480789/Put/seqid=0 2024-11-19T21:41:21,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742191_1367 (size=12301) 2024-11-19T21:41:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T21:41:21,309 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:21,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48100 deadline: 1732052541428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:21,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48048 deadline: 1732052541439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:21,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48090 deadline: 1732052541439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:41:21,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/fe4cce5290b2411ab69ec8463225dceb 2024-11-19T21:41:21,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/f33a4985f24f46f6a234ee271a2b1e0b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f33a4985f24f46f6a234ee271a2b1e0b 2024-11-19T21:41:21,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f33a4985f24f46f6a234ee271a2b1e0b, entries=200, sequenceid=493, filesize=14.4 K 2024-11-19T21:41:21,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/7feb02de4533455b96b1695436776e56 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7feb02de4533455b96b1695436776e56 2024-11-19T21:41:21,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7feb02de4533455b96b1695436776e56, entries=150, sequenceid=493, filesize=12.0 K 2024-11-19T21:41:21,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/fe4cce5290b2411ab69ec8463225dceb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fe4cce5290b2411ab69ec8463225dceb 2024-11-19T21:41:21,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fe4cce5290b2411ab69ec8463225dceb, entries=150, sequenceid=493, filesize=12.0 K 2024-11-19T21:41:21,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d5c69b89f3a83a34d8bee2d0fed83b40 in 912ms, sequenceid=493, compaction requested=true 2024-11-19T21:41:21,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:21,702 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5c69b89f3a83a34d8bee2d0fed83b40:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:21,702 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:21,702 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:21,702 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:21,703 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/A is initiating minor compaction (all files) 2024-11-19T21:41:21,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/B is initiating minor compaction (all files) 2024-11-19T21:41:21,703 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/A in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:21,703 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/B in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:21,703 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/dc71d0511f8f442db72e81fbc6d3d45b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/05894cfdc56841be87be4005f1fb86d0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f33a4985f24f46f6a234ee271a2b1e0b] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=39.5 K 2024-11-19T21:41:21,703 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2d2d0a3a475f42168830c81c322aad40, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4c94d8b432bc4282957d1aafd60c8142, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7feb02de4533455b96b1695436776e56] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=37.1 K 2024-11-19T21:41:21,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d2d0a3a475f42168830c81c322aad40, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732052477299 2024-11-19T21:41:21,703 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc71d0511f8f442db72e81fbc6d3d45b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732052477299 2024-11-19T21:41:21,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c94d8b432bc4282957d1aafd60c8142, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732052478462 2024-11-19T21:41:21,703 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05894cfdc56841be87be4005f1fb86d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732052478462 2024-11-19T21:41:21,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 7feb02de4533455b96b1695436776e56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732052479648 2024-11-19T21:41:21,703 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting f33a4985f24f46f6a234ee271a2b1e0b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732052479648 2024-11-19T21:41:21,709 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:21,709 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#A#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:21,709 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/57e02ccb2a51480ea11086e80d2ee784 is 50, key is test_row_0/B:col10/1732052480789/Put/seqid=0 2024-11-19T21:41:21,709 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/61d1da233ddc49d4a051d1d27cc3c580 is 50, key is test_row_0/A:col10/1732052480789/Put/seqid=0 2024-11-19T21:41:21,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742193_1369 (size=13493) 2024-11-19T21:41:21,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742192_1368 (size=13493) 2024-11-19T21:41:21,718 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/57e02ccb2a51480ea11086e80d2ee784 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/57e02ccb2a51480ea11086e80d2ee784 2024-11-19T21:41:21,722 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/B of d5c69b89f3a83a34d8bee2d0fed83b40 into 57e02ccb2a51480ea11086e80d2ee784(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:21,722 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:21,722 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/B, priority=13, startTime=1732052481702; duration=0sec 2024-11-19T21:41:21,723 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:21,723 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:B 2024-11-19T21:41:21,723 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:21,723 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:21,723 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): d5c69b89f3a83a34d8bee2d0fed83b40/C is initiating minor compaction (all files) 2024-11-19T21:41:21,724 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5c69b89f3a83a34d8bee2d0fed83b40/C in TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:21,724 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d131b0f902dc4b5b9adf8d8c363d1917, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c1eae4a3716841dc855e0e7ed7feecfe, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fe4cce5290b2411ab69ec8463225dceb] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp, totalSize=37.1 K 2024-11-19T21:41:21,724 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d131b0f902dc4b5b9adf8d8c363d1917, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732052477299 2024-11-19T21:41:21,724 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c1eae4a3716841dc855e0e7ed7feecfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732052478462 2024-11-19T21:41:21,725 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting fe4cce5290b2411ab69ec8463225dceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732052479648 2024-11-19T21:41:21,730 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5c69b89f3a83a34d8bee2d0fed83b40#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:21,731 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/2240347f186f48b2946d26980b6cac93 is 50, key is test_row_0/C:col10/1732052480789/Put/seqid=0 2024-11-19T21:41:21,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742194_1370 (size=13493) 2024-11-19T21:41:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:21,940 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:21,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T21:41:21,940 INFO [Thread-1208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1208. 2024-11-19T21:41:21,940 DEBUG [Thread-1208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:21,940 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:21,940 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:21,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:21,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/8127463729574f218b60feff7fb961b4 is 50, key is test_row_0/A:col10/1732052480800/Put/seqid=0 2024-11-19T21:41:21,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742195_1371 (size=12301) 2024-11-19T21:41:21,950 INFO [Thread-1212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1212. 2024-11-19T21:41:21,950 INFO [Thread-1206 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1206. 2024-11-19T21:41:21,950 DEBUG [Thread-1212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:21,950 DEBUG [Thread-1206 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:21,950 DEBUG [Thread-1212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:21,950 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:21,950 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:21,950 DEBUG [Thread-1212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:21,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:21,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:22,116 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/61d1da233ddc49d4a051d1d27cc3c580 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/61d1da233ddc49d4a051d1d27cc3c580 2024-11-19T21:41:22,121 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/A of d5c69b89f3a83a34d8bee2d0fed83b40 into 61d1da233ddc49d4a051d1d27cc3c580(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:22,121 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:22,121 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/A, priority=13, startTime=1732052481702; duration=0sec 2024-11-19T21:41:22,121 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:22,121 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:A 2024-11-19T21:41:22,138 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/2240347f186f48b2946d26980b6cac93 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2240347f186f48b2946d26980b6cac93 2024-11-19T21:41:22,143 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5c69b89f3a83a34d8bee2d0fed83b40/C of d5c69b89f3a83a34d8bee2d0fed83b40 into 2240347f186f48b2946d26980b6cac93(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:22,143 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:22,143 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., storeName=d5c69b89f3a83a34d8bee2d0fed83b40/C, priority=13, startTime=1732052481702; duration=0sec 2024-11-19T21:41:22,143 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:22,143 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5c69b89f3a83a34d8bee2d0fed83b40:C 2024-11-19T21:41:22,277 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T21:41:22,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/8127463729574f218b60feff7fb961b4 2024-11-19T21:41:22,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/02b961d6f2f546169af8b2ad38f9b358 is 50, key is test_row_0/B:col10/1732052480800/Put/seqid=0 2024-11-19T21:41:22,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742196_1372 (size=12301) 2024-11-19T21:41:22,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/02b961d6f2f546169af8b2ad38f9b358 2024-11-19T21:41:22,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/6d6039d0e1694ae8b315fd5ea1d72bcb is 50, key is test_row_0/C:col10/1732052480800/Put/seqid=0 2024-11-19T21:41:22,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742197_1373 (size=12301) 2024-11-19T21:41:23,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/6d6039d0e1694ae8b315fd5ea1d72bcb 2024-11-19T21:41:23,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/8127463729574f218b60feff7fb961b4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/8127463729574f218b60feff7fb961b4 2024-11-19T21:41:23,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/8127463729574f218b60feff7fb961b4, entries=150, sequenceid=508, filesize=12.0 K 2024-11-19T21:41:23,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/02b961d6f2f546169af8b2ad38f9b358 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/02b961d6f2f546169af8b2ad38f9b358 2024-11-19T21:41:23,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/02b961d6f2f546169af8b2ad38f9b358, entries=150, sequenceid=508, filesize=12.0 K 2024-11-19T21:41:23,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/6d6039d0e1694ae8b315fd5ea1d72bcb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d6039d0e1694ae8b315fd5ea1d72bcb 2024-11-19T21:41:23,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d6039d0e1694ae8b315fd5ea1d72bcb, entries=150, sequenceid=508, filesize=12.0 K 2024-11-19T21:41:23,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=13.42 KB/13740 for d5c69b89f3a83a34d8bee2d0fed83b40 in 1252ms, sequenceid=508, compaction requested=false 2024-11-19T21:41:23,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5c69b89f3a83a34d8bee2d0fed83b40: 2024-11-19T21:41:31,002 INFO [Thread-1210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1210. 2024-11-19T21:41:31,002 DEBUG [Thread-1210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:31,002 DEBUG [Thread-1210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:31,002 DEBUG [Thread-1210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:31,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:31,041 INFO [Thread-1214 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1214. 2024-11-19T21:41:31,041 DEBUG [Thread-1214 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:31,041 DEBUG [Thread-1214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:31,041 DEBUG [Thread-1214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:31,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:31,041 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3880 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3733 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3724 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3877 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3723 2024-11-19T21:41:31,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T21:41:31,042 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:41:31,043 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:41:31,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:31,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:31,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:31,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T21:41:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T21:41:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T21:41:31,050 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052491050"}]},"ts":"1732052491050"} 2024-11-19T21:41:31,052 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T21:41:31,052 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T21:41:31,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T21:41:31,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5c69b89f3a83a34d8bee2d0fed83b40, UNASSIGN}] 2024-11-19T21:41:31,054 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5c69b89f3a83a34d8bee2d0fed83b40, UNASSIGN 2024-11-19T21:41:31,055 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d5c69b89f3a83a34d8bee2d0fed83b40, regionState=CLOSING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:41:31,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5c69b89f3a83a34d8bee2d0fed83b40, UNASSIGN because future has completed 2024-11-19T21:41:31,056 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T21:41:31,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:41:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T21:41:31,209 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:31,210 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T21:41:31,210 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing d5c69b89f3a83a34d8bee2d0fed83b40, disabling compactions & flushes 2024-11-19T21:41:31,210 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:31,210 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:31,210 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. after waiting 0 ms 2024-11-19T21:41:31,210 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:31,210 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing d5c69b89f3a83a34d8bee2d0fed83b40 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-19T21:41:31,215 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/a2735d8019e443b889987586883fd9e4 is 50, key is test_row_0/A:col10/1732052481949/Put/seqid=0 2024-11-19T21:41:31,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742198_1374 (size=12301) 2024-11-19T21:41:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T21:41:31,621 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/a2735d8019e443b889987586883fd9e4 2024-11-19T21:41:31,634 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/d4f319b0e66c40b0a04fa47d0c2f2370 is 50, key is test_row_0/B:col10/1732052481949/Put/seqid=0 2024-11-19T21:41:31,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742199_1375 (size=12301) 2024-11-19T21:41:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T21:41:32,039 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/d4f319b0e66c40b0a04fa47d0c2f2370 2024-11-19T21:41:32,053 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/b866274d3d2244eaa81cf312588fcc3b is 50, key is test_row_0/C:col10/1732052481949/Put/seqid=0 2024-11-19T21:41:32,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742200_1376 (size=12301) 2024-11-19T21:41:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T21:41:32,458 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/b866274d3d2244eaa81cf312588fcc3b 2024-11-19T21:41:32,468 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/A/a2735d8019e443b889987586883fd9e4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a2735d8019e443b889987586883fd9e4 2024-11-19T21:41:32,474 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a2735d8019e443b889987586883fd9e4, entries=150, sequenceid=517, filesize=12.0 K 2024-11-19T21:41:32,475 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/B/d4f319b0e66c40b0a04fa47d0c2f2370 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/d4f319b0e66c40b0a04fa47d0c2f2370 2024-11-19T21:41:32,479 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/d4f319b0e66c40b0a04fa47d0c2f2370, entries=150, sequenceid=517, filesize=12.0 K 2024-11-19T21:41:32,480 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/.tmp/C/b866274d3d2244eaa81cf312588fcc3b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b866274d3d2244eaa81cf312588fcc3b 2024-11-19T21:41:32,483 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b866274d3d2244eaa81cf312588fcc3b, entries=150, sequenceid=517, filesize=12.0 K 2024-11-19T21:41:32,484 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d5c69b89f3a83a34d8bee2d0fed83b40 in 1273ms, sequenceid=517, compaction requested=true 2024-11-19T21:41:32,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/fe7b28f39986468c926cb8123f4d8487, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/999539e530184d3584b8e99661e8cb97, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/eee57d0ca2424d94915c1d39293a23e1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/b4ea200ddffc469abf4a1b7d371f6845, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/08322169dac3431fb579d0f4b2180c7c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/79f2294723b3443a85a8b74a385d730f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1c4ae2c3fddf4ff0a3b4f767625d2aae, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/e6941b868ff1412288cab18a8892df44, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/23c5e4c4e6454654a6f197bf04338c32, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1bae88889f9446e0b5fe9337cb47ca9a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d82bc4943b2148148451cfe8758dce61, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a16e909541204b67bbeea473e4764893, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/89cc08e367db4b30bddc2b6ccd57da56, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/01845cae3f074f419b730c3b74735eac, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1a7bfb0656c94c8fb701b1b137c98d0f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/819e0da8d11a4b90a453bc03c4765f57, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f198f009a7554fb4ab0291395628c164, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c30b6e25b5684879b026d1b5d76facb2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/99931e6f1bf84b49a88d94f4dc195f0b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/9dfd73c8a57d48eb9f7ef9edece250ea, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d190636a575d4e57ae28bf895b9ec333, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/21259ac11aa2472cb9f6f3e3ac619036, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/16fff2ec3df7404aa26ac1cc429548bb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7149a90dd9ad48429b2b011e31f450be, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/2bc6b95b14604d58b542c847cbd37e59, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/12f6c56caf8d4e83873637087e520470, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7297424da29f40ed8ffb1ecd5b47d4fd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f54530beab9b4a798a62b77421228d0c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c22c155083a64269baba22618214e9e1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/8069287b9f664f819d63ea61fc1454d6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f06c30d061d340d1acafecf1538a8007, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/54620dc4c69149d0801e450c9d4e0a40, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/dc71d0511f8f442db72e81fbc6d3d45b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/05894cfdc56841be87be4005f1fb86d0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f33a4985f24f46f6a234ee271a2b1e0b] to archive 2024-11-19T21:41:32,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:41:32,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/fe7b28f39986468c926cb8123f4d8487 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/fe7b28f39986468c926cb8123f4d8487 2024-11-19T21:41:32,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/999539e530184d3584b8e99661e8cb97 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/999539e530184d3584b8e99661e8cb97 2024-11-19T21:41:32,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/eee57d0ca2424d94915c1d39293a23e1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/eee57d0ca2424d94915c1d39293a23e1 2024-11-19T21:41:32,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/b4ea200ddffc469abf4a1b7d371f6845 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/b4ea200ddffc469abf4a1b7d371f6845 2024-11-19T21:41:32,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/08322169dac3431fb579d0f4b2180c7c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/08322169dac3431fb579d0f4b2180c7c 2024-11-19T21:41:32,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/79f2294723b3443a85a8b74a385d730f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/79f2294723b3443a85a8b74a385d730f 2024-11-19T21:41:32,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1c4ae2c3fddf4ff0a3b4f767625d2aae to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1c4ae2c3fddf4ff0a3b4f767625d2aae 2024-11-19T21:41:32,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/e6941b868ff1412288cab18a8892df44 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/e6941b868ff1412288cab18a8892df44 2024-11-19T21:41:32,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/23c5e4c4e6454654a6f197bf04338c32 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/23c5e4c4e6454654a6f197bf04338c32 2024-11-19T21:41:32,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1bae88889f9446e0b5fe9337cb47ca9a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1bae88889f9446e0b5fe9337cb47ca9a 2024-11-19T21:41:32,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d82bc4943b2148148451cfe8758dce61 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d82bc4943b2148148451cfe8758dce61 2024-11-19T21:41:32,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a16e909541204b67bbeea473e4764893 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a16e909541204b67bbeea473e4764893 2024-11-19T21:41:32,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/89cc08e367db4b30bddc2b6ccd57da56 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/89cc08e367db4b30bddc2b6ccd57da56 2024-11-19T21:41:32,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/01845cae3f074f419b730c3b74735eac to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/01845cae3f074f419b730c3b74735eac 2024-11-19T21:41:32,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1a7bfb0656c94c8fb701b1b137c98d0f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/1a7bfb0656c94c8fb701b1b137c98d0f 2024-11-19T21:41:32,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/819e0da8d11a4b90a453bc03c4765f57 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/819e0da8d11a4b90a453bc03c4765f57 2024-11-19T21:41:32,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f198f009a7554fb4ab0291395628c164 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f198f009a7554fb4ab0291395628c164 2024-11-19T21:41:32,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c30b6e25b5684879b026d1b5d76facb2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c30b6e25b5684879b026d1b5d76facb2 2024-11-19T21:41:32,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/99931e6f1bf84b49a88d94f4dc195f0b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/99931e6f1bf84b49a88d94f4dc195f0b 2024-11-19T21:41:32,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/9dfd73c8a57d48eb9f7ef9edece250ea to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/9dfd73c8a57d48eb9f7ef9edece250ea 2024-11-19T21:41:32,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d190636a575d4e57ae28bf895b9ec333 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/d190636a575d4e57ae28bf895b9ec333 2024-11-19T21:41:32,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/21259ac11aa2472cb9f6f3e3ac619036 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/21259ac11aa2472cb9f6f3e3ac619036 2024-11-19T21:41:32,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/16fff2ec3df7404aa26ac1cc429548bb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/16fff2ec3df7404aa26ac1cc429548bb 2024-11-19T21:41:32,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7149a90dd9ad48429b2b011e31f450be to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7149a90dd9ad48429b2b011e31f450be 2024-11-19T21:41:32,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/2bc6b95b14604d58b542c847cbd37e59 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/2bc6b95b14604d58b542c847cbd37e59 2024-11-19T21:41:32,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/12f6c56caf8d4e83873637087e520470 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/12f6c56caf8d4e83873637087e520470 2024-11-19T21:41:32,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7297424da29f40ed8ffb1ecd5b47d4fd to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/7297424da29f40ed8ffb1ecd5b47d4fd 2024-11-19T21:41:32,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f54530beab9b4a798a62b77421228d0c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f54530beab9b4a798a62b77421228d0c 2024-11-19T21:41:32,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c22c155083a64269baba22618214e9e1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/c22c155083a64269baba22618214e9e1 2024-11-19T21:41:32,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/8069287b9f664f819d63ea61fc1454d6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/8069287b9f664f819d63ea61fc1454d6 2024-11-19T21:41:32,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f06c30d061d340d1acafecf1538a8007 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f06c30d061d340d1acafecf1538a8007 2024-11-19T21:41:32,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/54620dc4c69149d0801e450c9d4e0a40 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/54620dc4c69149d0801e450c9d4e0a40 2024-11-19T21:41:32,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/dc71d0511f8f442db72e81fbc6d3d45b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/dc71d0511f8f442db72e81fbc6d3d45b 2024-11-19T21:41:32,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/05894cfdc56841be87be4005f1fb86d0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/05894cfdc56841be87be4005f1fb86d0 2024-11-19T21:41:32,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f33a4985f24f46f6a234ee271a2b1e0b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/f33a4985f24f46f6a234ee271a2b1e0b 2024-11-19T21:41:32,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/5c4af768a7f2424499a17c296f3cf219, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7e078fbe33064c989590fb5d2d37709c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4737dfcf2dae433ab7cb34f85e00341e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7d279bb86c5d499c9932f805a796e1fd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/eec26297e9714ada93b53b5ae08336e6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/3dfc7e64d2ee405d85fe3fc772890441, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/38694a03679e444bba93ceee157e3f15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f0ea9d7815e7484987f675515c64b4d7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e1ca06f5835047ff8070d46cd07b7afe, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/414e707967984abc81245ac158aac613, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/cfe6741370434b8bac99b241e08fc376, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/09c98888c7f347afac9d9180e36d80af, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/af0fcb8de1aa40878698be533547f1d5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ee6bb7c92a54471983481e3b6cd28542, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/b9dac01325224bf1b9aebc3696947936, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/0f8fe8ebe267471daebb364e34afa99d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/01b91f291e1849a3aea33c9ac156fe4c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/25d66a0803554ac4b59ddd6c0347f045, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2e29aed923874702a1ffe4fa3bdb0ee7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f62d125d1c6140af98b024e71bf25b9c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f921280b1c3745069db91df1d359a245, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7af69dffe58f42eabab522b7b92d1de1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/22174094fccd48a5aab489af10d0c567, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4a90302483dc43d3aca850965f23622f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/c303397e984d434e963c569852ffdf46, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/efe0b6e95b6047e9bcffb354c1ce4aa4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/13594de9bc8a4865b90a74fd713b5ce3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e18f958d5ed94c7e8f3858762c091d79, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/430028ee05c74086b4c5d9242de217d7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/10aeff4796fd4c65a91f08285f42a351, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ec5b8b5d92854cf4b2080ebe985e7f18, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2d2d0a3a475f42168830c81c322aad40, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/56cc94aca8ef4934b6a087d3b064e0eb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4c94d8b432bc4282957d1aafd60c8142, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7feb02de4533455b96b1695436776e56] to archive 2024-11-19T21:41:32,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:41:32,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/5c4af768a7f2424499a17c296f3cf219 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/5c4af768a7f2424499a17c296f3cf219 2024-11-19T21:41:32,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7e078fbe33064c989590fb5d2d37709c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7e078fbe33064c989590fb5d2d37709c 2024-11-19T21:41:32,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4737dfcf2dae433ab7cb34f85e00341e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4737dfcf2dae433ab7cb34f85e00341e 2024-11-19T21:41:32,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7d279bb86c5d499c9932f805a796e1fd to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7d279bb86c5d499c9932f805a796e1fd 2024-11-19T21:41:32,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/eec26297e9714ada93b53b5ae08336e6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/eec26297e9714ada93b53b5ae08336e6 2024-11-19T21:41:32,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/3dfc7e64d2ee405d85fe3fc772890441 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/3dfc7e64d2ee405d85fe3fc772890441 2024-11-19T21:41:32,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/38694a03679e444bba93ceee157e3f15 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/38694a03679e444bba93ceee157e3f15 2024-11-19T21:41:32,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f0ea9d7815e7484987f675515c64b4d7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f0ea9d7815e7484987f675515c64b4d7 2024-11-19T21:41:32,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e1ca06f5835047ff8070d46cd07b7afe to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e1ca06f5835047ff8070d46cd07b7afe 2024-11-19T21:41:32,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/414e707967984abc81245ac158aac613 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/414e707967984abc81245ac158aac613 2024-11-19T21:41:32,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/cfe6741370434b8bac99b241e08fc376 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/cfe6741370434b8bac99b241e08fc376 2024-11-19T21:41:32,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/09c98888c7f347afac9d9180e36d80af to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/09c98888c7f347afac9d9180e36d80af 2024-11-19T21:41:32,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/af0fcb8de1aa40878698be533547f1d5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/af0fcb8de1aa40878698be533547f1d5 2024-11-19T21:41:32,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ee6bb7c92a54471983481e3b6cd28542 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ee6bb7c92a54471983481e3b6cd28542 2024-11-19T21:41:32,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/b9dac01325224bf1b9aebc3696947936 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/b9dac01325224bf1b9aebc3696947936 2024-11-19T21:41:32,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/0f8fe8ebe267471daebb364e34afa99d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/0f8fe8ebe267471daebb364e34afa99d 2024-11-19T21:41:32,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/01b91f291e1849a3aea33c9ac156fe4c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/01b91f291e1849a3aea33c9ac156fe4c 2024-11-19T21:41:32,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/25d66a0803554ac4b59ddd6c0347f045 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/25d66a0803554ac4b59ddd6c0347f045 2024-11-19T21:41:32,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2e29aed923874702a1ffe4fa3bdb0ee7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2e29aed923874702a1ffe4fa3bdb0ee7 2024-11-19T21:41:32,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f62d125d1c6140af98b024e71bf25b9c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f62d125d1c6140af98b024e71bf25b9c 2024-11-19T21:41:32,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f921280b1c3745069db91df1d359a245 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/f921280b1c3745069db91df1d359a245 2024-11-19T21:41:32,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7af69dffe58f42eabab522b7b92d1de1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7af69dffe58f42eabab522b7b92d1de1 2024-11-19T21:41:32,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/22174094fccd48a5aab489af10d0c567 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/22174094fccd48a5aab489af10d0c567 2024-11-19T21:41:32,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4a90302483dc43d3aca850965f23622f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4a90302483dc43d3aca850965f23622f 2024-11-19T21:41:32,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/c303397e984d434e963c569852ffdf46 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/c303397e984d434e963c569852ffdf46 2024-11-19T21:41:32,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/efe0b6e95b6047e9bcffb354c1ce4aa4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/efe0b6e95b6047e9bcffb354c1ce4aa4 2024-11-19T21:41:32,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/13594de9bc8a4865b90a74fd713b5ce3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/13594de9bc8a4865b90a74fd713b5ce3 2024-11-19T21:41:32,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e18f958d5ed94c7e8f3858762c091d79 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/e18f958d5ed94c7e8f3858762c091d79 2024-11-19T21:41:32,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/430028ee05c74086b4c5d9242de217d7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/430028ee05c74086b4c5d9242de217d7 2024-11-19T21:41:32,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/10aeff4796fd4c65a91f08285f42a351 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/10aeff4796fd4c65a91f08285f42a351 2024-11-19T21:41:32,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ec5b8b5d92854cf4b2080ebe985e7f18 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/ec5b8b5d92854cf4b2080ebe985e7f18 2024-11-19T21:41:32,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2d2d0a3a475f42168830c81c322aad40 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/2d2d0a3a475f42168830c81c322aad40 2024-11-19T21:41:32,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/56cc94aca8ef4934b6a087d3b064e0eb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/56cc94aca8ef4934b6a087d3b064e0eb 2024-11-19T21:41:32,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4c94d8b432bc4282957d1aafd60c8142 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/4c94d8b432bc4282957d1aafd60c8142 2024-11-19T21:41:32,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7feb02de4533455b96b1695436776e56 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/7feb02de4533455b96b1695436776e56 2024-11-19T21:41:32,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e6c13d419e647dcafa4d0d1f3f57cf4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d666cad349a451c832bef70157da3f3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2ca725f8269c441bb985336decc9d57e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a7ba88f41e354a2e9e4e818176eca1e6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2e680e7a08ec4eb1adfdd8f1ea37a423, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0c63de8912a546afb3a9046d731063bf, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fb3d1934ce74442d9a8557c6b38c47fb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/65d94c898c9d41559053dec0e0a2a2ba, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8a6e8ed500e04bdd8c8ccb924f1fdae7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b878c34115b848b381efef239a96d6ce, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/90fa0072684a4a3487e737624b092165, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/3e99a26441dd4b3fb53feb3a305fd8ba, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0046f43fa29c4e83ada0ff7cf057df4d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/331105c72a3e4554a29e06801b5f628e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d3db249bbfd747708e8f7bdc452f68f1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/cf89669aa116484cbbf9567bc2b17f09, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/398ab7fef2b94d39adbe952b40b85a15, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a96335f9bf2c4297be51d864a830266a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8ce40b590ab84bf79185acbae79cedee, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/e59c786d9b564beab339d931f30c8f03, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/acb50c7007344ff3b738211eee8fa73f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/50c86bbb504148eab690078d2fd39d98, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d030a9b89b184a9fa64cbeda1c038353, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0cb0358a69a14d92906c83757f46eea8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c82eadb1877a4d4583e4e256f9444ebb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/853233576f6e4904b0d79f0b4f60bc74, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0060a4b42ed24fa3bec38322e5bad795, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/dca2a05fc34247798036b20e9829d146, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/80b2463607374d1aa7d444cc371ad879, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e9d90013b61406fa8b5d126f425bc16, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/03051d0db2494e0d8a7b1fba3f3c0cec, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d131b0f902dc4b5b9adf8d8c363d1917, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/ad06c7f2511e4b2aae4176b3e81673bd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c1eae4a3716841dc855e0e7ed7feecfe, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fe4cce5290b2411ab69ec8463225dceb] to archive 2024-11-19T21:41:32,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:41:32,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e6c13d419e647dcafa4d0d1f3f57cf4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e6c13d419e647dcafa4d0d1f3f57cf4 2024-11-19T21:41:32,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d666cad349a451c832bef70157da3f3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d666cad349a451c832bef70157da3f3 2024-11-19T21:41:32,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2ca725f8269c441bb985336decc9d57e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2ca725f8269c441bb985336decc9d57e 2024-11-19T21:41:32,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a7ba88f41e354a2e9e4e818176eca1e6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a7ba88f41e354a2e9e4e818176eca1e6 2024-11-19T21:41:32,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2e680e7a08ec4eb1adfdd8f1ea37a423 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2e680e7a08ec4eb1adfdd8f1ea37a423 2024-11-19T21:41:32,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0c63de8912a546afb3a9046d731063bf to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0c63de8912a546afb3a9046d731063bf 2024-11-19T21:41:32,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fb3d1934ce74442d9a8557c6b38c47fb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fb3d1934ce74442d9a8557c6b38c47fb 2024-11-19T21:41:32,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/65d94c898c9d41559053dec0e0a2a2ba to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/65d94c898c9d41559053dec0e0a2a2ba 2024-11-19T21:41:32,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8a6e8ed500e04bdd8c8ccb924f1fdae7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8a6e8ed500e04bdd8c8ccb924f1fdae7 2024-11-19T21:41:32,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b878c34115b848b381efef239a96d6ce to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b878c34115b848b381efef239a96d6ce 2024-11-19T21:41:32,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/90fa0072684a4a3487e737624b092165 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/90fa0072684a4a3487e737624b092165 2024-11-19T21:41:32,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/3e99a26441dd4b3fb53feb3a305fd8ba to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/3e99a26441dd4b3fb53feb3a305fd8ba 2024-11-19T21:41:32,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0046f43fa29c4e83ada0ff7cf057df4d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0046f43fa29c4e83ada0ff7cf057df4d 2024-11-19T21:41:32,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/331105c72a3e4554a29e06801b5f628e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/331105c72a3e4554a29e06801b5f628e 2024-11-19T21:41:32,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d3db249bbfd747708e8f7bdc452f68f1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d3db249bbfd747708e8f7bdc452f68f1 2024-11-19T21:41:32,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/cf89669aa116484cbbf9567bc2b17f09 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/cf89669aa116484cbbf9567bc2b17f09 2024-11-19T21:41:32,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/398ab7fef2b94d39adbe952b40b85a15 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/398ab7fef2b94d39adbe952b40b85a15 2024-11-19T21:41:32,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a96335f9bf2c4297be51d864a830266a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/a96335f9bf2c4297be51d864a830266a 2024-11-19T21:41:32,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8ce40b590ab84bf79185acbae79cedee to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/8ce40b590ab84bf79185acbae79cedee 2024-11-19T21:41:32,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/e59c786d9b564beab339d931f30c8f03 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/e59c786d9b564beab339d931f30c8f03 2024-11-19T21:41:32,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/acb50c7007344ff3b738211eee8fa73f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/acb50c7007344ff3b738211eee8fa73f 2024-11-19T21:41:32,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/50c86bbb504148eab690078d2fd39d98 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/50c86bbb504148eab690078d2fd39d98 2024-11-19T21:41:32,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d030a9b89b184a9fa64cbeda1c038353 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d030a9b89b184a9fa64cbeda1c038353 2024-11-19T21:41:32,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0cb0358a69a14d92906c83757f46eea8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0cb0358a69a14d92906c83757f46eea8 2024-11-19T21:41:32,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c82eadb1877a4d4583e4e256f9444ebb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c82eadb1877a4d4583e4e256f9444ebb 2024-11-19T21:41:32,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/853233576f6e4904b0d79f0b4f60bc74 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/853233576f6e4904b0d79f0b4f60bc74 2024-11-19T21:41:32,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0060a4b42ed24fa3bec38322e5bad795 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/0060a4b42ed24fa3bec38322e5bad795 2024-11-19T21:41:32,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/dca2a05fc34247798036b20e9829d146 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/dca2a05fc34247798036b20e9829d146 2024-11-19T21:41:32,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/80b2463607374d1aa7d444cc371ad879 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/80b2463607374d1aa7d444cc371ad879 2024-11-19T21:41:32,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e9d90013b61406fa8b5d126f425bc16 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/7e9d90013b61406fa8b5d126f425bc16 2024-11-19T21:41:32,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/03051d0db2494e0d8a7b1fba3f3c0cec to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/03051d0db2494e0d8a7b1fba3f3c0cec 2024-11-19T21:41:32,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d131b0f902dc4b5b9adf8d8c363d1917 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/d131b0f902dc4b5b9adf8d8c363d1917 2024-11-19T21:41:32,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/ad06c7f2511e4b2aae4176b3e81673bd to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/ad06c7f2511e4b2aae4176b3e81673bd 2024-11-19T21:41:32,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c1eae4a3716841dc855e0e7ed7feecfe to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/c1eae4a3716841dc855e0e7ed7feecfe 2024-11-19T21:41:32,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fe4cce5290b2411ab69ec8463225dceb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/fe4cce5290b2411ab69ec8463225dceb 2024-11-19T21:41:32,598 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/recovered.edits/520.seqid, newMaxSeqId=520, maxSeqId=1 2024-11-19T21:41:32,599 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. 2024-11-19T21:41:32,599 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for d5c69b89f3a83a34d8bee2d0fed83b40: Waiting for close lock at 1732052491210Running coprocessor pre-close hooks at 1732052491210Disabling compacts and flushes for region at 1732052491210Disabling writes for close at 1732052491210Obtaining lock to block concurrent updates at 1732052491210Preparing flush snapshotting stores in d5c69b89f3a83a34d8bee2d0fed83b40 at 1732052491210Finished memstore snapshotting TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732052491211 (+1 ms)Flushing stores of TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40. at 1732052491212 (+1 ms)Flushing d5c69b89f3a83a34d8bee2d0fed83b40/A: creating writer at 1732052491212Flushing d5c69b89f3a83a34d8bee2d0fed83b40/A: appending metadata at 1732052491215 (+3 ms)Flushing d5c69b89f3a83a34d8bee2d0fed83b40/A: closing flushed file at 1732052491215Flushing d5c69b89f3a83a34d8bee2d0fed83b40/B: creating writer at 1732052491631 (+416 ms)Flushing d5c69b89f3a83a34d8bee2d0fed83b40/B: appending metadata at 1732052491634 (+3 ms)Flushing d5c69b89f3a83a34d8bee2d0fed83b40/B: closing flushed file at 1732052491634Flushing d5c69b89f3a83a34d8bee2d0fed83b40/C: creating writer at 1732052492048 (+414 ms)Flushing d5c69b89f3a83a34d8bee2d0fed83b40/C: appending metadata at 1732052492053 (+5 ms)Flushing d5c69b89f3a83a34d8bee2d0fed83b40/C: closing flushed file at 1732052492053Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b0fec4f: reopening flushed file at 1732052492467 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4471e5d7: reopening flushed file at 1732052492474 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d538a82: reopening flushed file at 1732052492479 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d5c69b89f3a83a34d8bee2d0fed83b40 in 1273ms, sequenceid=517, compaction requested=true at 1732052492484 (+5 ms)Writing region close event to WAL at 1732052492595 (+111 ms)Running coprocessor post-close hooks at 1732052492599 (+4 ms)Closed at 1732052492599 2024-11-19T21:41:32,601 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:32,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d5c69b89f3a83a34d8bee2d0fed83b40, regionState=CLOSED 2024-11-19T21:41:32,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:41:32,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-19T21:41:32,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure d5c69b89f3a83a34d8bee2d0fed83b40, server=707262ffa86d,42821,1732052394005 in 1.5510 sec 2024-11-19T21:41:32,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-19T21:41:32,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5c69b89f3a83a34d8bee2d0fed83b40, UNASSIGN in 1.5560 sec 2024-11-19T21:41:32,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T21:41:32,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5590 sec 2024-11-19T21:41:32,614 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052492614"}]},"ts":"1732052492614"} 2024-11-19T21:41:32,615 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T21:41:32,615 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T21:41:32,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5710 sec 2024-11-19T21:41:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T21:41:33,210 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:33,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T21:41:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:41:33,216 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:41:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T21:41:33,218 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:41:33,224 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:33,228 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/recovered.edits] 2024-11-19T21:41:33,232 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/61d1da233ddc49d4a051d1d27cc3c580 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/61d1da233ddc49d4a051d1d27cc3c580 2024-11-19T21:41:33,234 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/8127463729574f218b60feff7fb961b4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/8127463729574f218b60feff7fb961b4 2024-11-19T21:41:33,236 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a2735d8019e443b889987586883fd9e4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/A/a2735d8019e443b889987586883fd9e4 2024-11-19T21:41:33,240 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/02b961d6f2f546169af8b2ad38f9b358 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/02b961d6f2f546169af8b2ad38f9b358 2024-11-19T21:41:33,242 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/57e02ccb2a51480ea11086e80d2ee784 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/57e02ccb2a51480ea11086e80d2ee784 2024-11-19T21:41:33,245 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/d4f319b0e66c40b0a04fa47d0c2f2370 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/B/d4f319b0e66c40b0a04fa47d0c2f2370 2024-11-19T21:41:33,249 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2240347f186f48b2946d26980b6cac93 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/2240347f186f48b2946d26980b6cac93 2024-11-19T21:41:33,251 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d6039d0e1694ae8b315fd5ea1d72bcb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/6d6039d0e1694ae8b315fd5ea1d72bcb 2024-11-19T21:41:33,253 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b866274d3d2244eaa81cf312588fcc3b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/C/b866274d3d2244eaa81cf312588fcc3b 2024-11-19T21:41:33,258 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/recovered.edits/520.seqid to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40/recovered.edits/520.seqid 2024-11-19T21:41:33,259 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/d5c69b89f3a83a34d8bee2d0fed83b40 2024-11-19T21:41:33,259 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T21:41:33,263 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:41:33,268 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T21:41:33,270 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T21:41:33,271 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:41:33,271 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T21:41:33,271 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732052493271"}]},"ts":"9223372036854775807"} 2024-11-19T21:41:33,273 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T21:41:33,273 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d5c69b89f3a83a34d8bee2d0fed83b40, NAME => 'TestAcidGuarantees,,1732052458745.d5c69b89f3a83a34d8bee2d0fed83b40.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T21:41:33,273 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T21:41:33,273 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732052493273"}]},"ts":"9223372036854775807"} 2024-11-19T21:41:33,275 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T21:41:33,276 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:41:33,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-19T21:41:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T21:41:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T21:41:33,329 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:33,338 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237 (was 239), OpenFileDescriptor=445 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=422 (was 507), ProcessCount=11 (was 11), AvailableMemoryMB=5071 (was 4670) - AvailableMemoryMB LEAK? - 2024-11-19T21:41:33,348 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=422, ProcessCount=11, AvailableMemoryMB=5071 2024-11-19T21:41:33,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T21:41:33,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:41:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T21:41:33,352 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T21:41:33,352 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:33,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-19T21:41:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T21:41:33,353 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T21:41:33,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742201_1377 (size=959) 2024-11-19T21:41:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T21:41:33,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T21:41:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T21:41:33,763 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b 2024-11-19T21:41:33,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742202_1378 (size=53) 2024-11-19T21:41:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T21:41:34,173 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:41:34,173 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a87877009cc80e5fd95e37cd9db304a4, disabling compactions & flushes 2024-11-19T21:41:34,173 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:34,174 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:34,174 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. after waiting 0 ms 2024-11-19T21:41:34,174 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:34,174 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:34,174 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a87877009cc80e5fd95e37cd9db304a4: Waiting for close lock at 1732052494173Disabling compacts and flushes for region at 1732052494173Disabling writes for close at 1732052494174 (+1 ms)Writing region close event to WAL at 1732052494174Closed at 1732052494174 2024-11-19T21:41:34,176 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T21:41:34,177 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732052494176"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732052494176"}]},"ts":"1732052494176"} 2024-11-19T21:41:34,179 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T21:41:34,180 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T21:41:34,180 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052494180"}]},"ts":"1732052494180"} 2024-11-19T21:41:34,182 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T21:41:34,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, ASSIGN}] 2024-11-19T21:41:34,183 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, ASSIGN 2024-11-19T21:41:34,185 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, ASSIGN; state=OFFLINE, location=707262ffa86d,42821,1732052394005; forceNewPlan=false, retain=false 2024-11-19T21:41:34,335 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a87877009cc80e5fd95e37cd9db304a4, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:41:34,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, ASSIGN because future has completed 2024-11-19T21:41:34,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:41:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T21:41:34,497 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:34,498 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:41:34,498 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,499 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:41:34,499 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,499 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,501 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,503 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a87877009cc80e5fd95e37cd9db304a4 columnFamilyName A 2024-11-19T21:41:34,503 DEBUG [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:34,504 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(327): Store=a87877009cc80e5fd95e37cd9db304a4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:41:34,504 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,505 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a87877009cc80e5fd95e37cd9db304a4 columnFamilyName B 2024-11-19T21:41:34,505 DEBUG [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:34,506 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(327): Store=a87877009cc80e5fd95e37cd9db304a4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:41:34,506 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,507 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a87877009cc80e5fd95e37cd9db304a4 columnFamilyName C 2024-11-19T21:41:34,507 DEBUG [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:34,508 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(327): Store=a87877009cc80e5fd95e37cd9db304a4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:41:34,508 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,509 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,509 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,511 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,511 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,512 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:41:34,513 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,517 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:41:34,517 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened a87877009cc80e5fd95e37cd9db304a4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63502861, jitterRate=-0.05373363196849823}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:41:34,518 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:34,519 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for a87877009cc80e5fd95e37cd9db304a4: Running coprocessor pre-open hook at 1732052494499Writing region info on filesystem at 1732052494499Initializing all the Stores at 1732052494501 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052494501Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052494501Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052494501Cleaning up temporary data from old regions at 1732052494511 (+10 ms)Running coprocessor post-open hooks at 1732052494518 (+7 ms)Region opened successfully at 1732052494518 2024-11-19T21:41:34,519 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., pid=91, masterSystemTime=1732052494490 2024-11-19T21:41:34,521 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:34,521 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:34,521 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a87877009cc80e5fd95e37cd9db304a4, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:41:34,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:41:34,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T21:41:34,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 in 185 msec 2024-11-19T21:41:34,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-19T21:41:34,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, ASSIGN in 342 msec 2024-11-19T21:41:34,526 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T21:41:34,527 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052494526"}]},"ts":"1732052494526"} 2024-11-19T21:41:34,528 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T21:41:34,528 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T21:41:34,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-19T21:41:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-19T21:41:35,510 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:35,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab2a0ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:35,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:35,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:35,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302fea40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:35,519 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4014306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:35,520 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:35,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:35,522 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T21:41:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 707262ffa86d,37541,1732052393169 2024-11-19T21:41:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@643d225b 2024-11-19T21:41:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T21:41:35,523 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T21:41:35,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T21:41:35,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:41:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T21:41:35,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742203_1379 (size=995) 2024-11-19T21:41:35,936 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T21:41:35,936 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T21:41:35,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T21:41:35,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, REOPEN/MOVE}] 2024-11-19T21:41:35,944 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, REOPEN/MOVE 2024-11-19T21:41:35,945 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a87877009cc80e5fd95e37cd9db304a4, regionState=CLOSING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:41:35,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, REOPEN/MOVE because future has completed 2024-11-19T21:41:35,947 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T21:41:35,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:41:36,100 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,101 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T21:41:36,101 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing a87877009cc80e5fd95e37cd9db304a4, disabling compactions & flushes 2024-11-19T21:41:36,101 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,101 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,101 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. after waiting 0 ms 2024-11-19T21:41:36,101 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,109 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T21:41:36,109 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,109 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for a87877009cc80e5fd95e37cd9db304a4: Waiting for close lock at 1732052496101Running coprocessor pre-close hooks at 1732052496101Disabling compacts and flushes for region at 1732052496101Disabling writes for close at 1732052496101Writing region close event to WAL at 1732052496102 (+1 ms)Running coprocessor post-close hooks at 1732052496109 (+7 ms)Closed at 1732052496109 2024-11-19T21:41:36,109 WARN [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: a87877009cc80e5fd95e37cd9db304a4 to self. 2024-11-19T21:41:36,111 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,112 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a87877009cc80e5fd95e37cd9db304a4, regionState=CLOSED 2024-11-19T21:41:36,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:41:36,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-19T21:41:36,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 in 167 msec 2024-11-19T21:41:36,117 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, REOPEN/MOVE; state=CLOSED, location=707262ffa86d,42821,1732052394005; forceNewPlan=false, retain=true 2024-11-19T21:41:36,268 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a87877009cc80e5fd95e37cd9db304a4, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, REOPEN/MOVE because future has completed 2024-11-19T21:41:36,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:41:36,428 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,428 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:41:36,429 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,429 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:41:36,429 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,430 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,433 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,435 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a87877009cc80e5fd95e37cd9db304a4 columnFamilyName A 2024-11-19T21:41:36,438 DEBUG [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:36,439 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(327): Store=a87877009cc80e5fd95e37cd9db304a4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:41:36,439 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,440 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a87877009cc80e5fd95e37cd9db304a4 columnFamilyName B 2024-11-19T21:41:36,440 DEBUG [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:36,440 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(327): Store=a87877009cc80e5fd95e37cd9db304a4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:41:36,441 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,441 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a87877009cc80e5fd95e37cd9db304a4 columnFamilyName C 2024-11-19T21:41:36,441 DEBUG [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:36,442 INFO [StoreOpener-a87877009cc80e5fd95e37cd9db304a4-1 {}] regionserver.HStore(327): Store=a87877009cc80e5fd95e37cd9db304a4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:41:36,442 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,442 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,443 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,444 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,444 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,445 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:41:36,446 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,447 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened a87877009cc80e5fd95e37cd9db304a4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75144068, jitterRate=0.11973387002944946}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:41:36,447 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,447 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for a87877009cc80e5fd95e37cd9db304a4: Running coprocessor pre-open hook at 1732052496430Writing region info on filesystem at 1732052496430Initializing all the Stores at 1732052496432 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052496432Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052496432Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052496432Cleaning up temporary data from old regions at 1732052496444 (+12 ms)Running coprocessor post-open hooks at 1732052496447 (+3 ms)Region opened successfully at 1732052496447 2024-11-19T21:41:36,448 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., pid=96, masterSystemTime=1732052496422 2024-11-19T21:41:36,450 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,450 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,450 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a87877009cc80e5fd95e37cd9db304a4, regionState=OPEN, openSeqNum=5, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:41:36,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-19T21:41:36,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 in 182 msec 2024-11-19T21:41:36,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T21:41:36,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, REOPEN/MOVE in 510 msec 2024-11-19T21:41:36,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-19T21:41:36,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-19T21:41:36,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 932 msec 2024-11-19T21:41:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T21:41:36,459 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:36,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38802cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,460 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ce973f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,461 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53aedd07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209ba84b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,463 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1843d20e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,464 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c53ba28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7495e76b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,466 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e585c1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,467 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58f3b7bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7288b4d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,468 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178aabed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,470 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58f076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69edc398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,471 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@714544be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,473 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20641e24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62931224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,475 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60781bce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,476 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b1c2ef0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400fc310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,478 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd04393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,479 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12404e46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227d6625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,481 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d2e058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7eff0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15113514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,485 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9d5a54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,486 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47cced20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ed8e2d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:41:36,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:41:36,489 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:41:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:41:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:41:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11597eac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:41:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:36,491 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:41:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35660f96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:41:36,492 DEBUG [Thread-1724 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-19T21:41:36,494 DEBUG [Thread-1707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T21:41:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,495 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:36,495 DEBUG [Thread-1726 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,495 DEBUG [Thread-1713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,496 DEBUG [Thread-1715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,496 DEBUG [Thread-1720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,496 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:36,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,502 DEBUG [Thread-1709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,503 DEBUG [Thread-1709 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,503 DEBUG [Thread-1709 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,504 DEBUG [Thread-1718 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,504 DEBUG [Thread-1722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,506 DEBUG [Thread-1711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:41:36,507 DEBUG [Thread-1718 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,507 DEBUG [Thread-1718 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,510 DEBUG [Thread-1711 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:41:36,511 DEBUG [Thread-1711 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:41:36,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:41:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,514 DEBUG [Thread-1718 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:41:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:36,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:41:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a2a50ffb6d0543cca4c1a2e5307b2c7b_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052496515/Put/seqid=0 2024-11-19T21:41:36,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052556542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052556547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052556549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052556550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052556552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742204_1380 (size=19474) 2024-11-19T21:41:36,566 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:36,572 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a2a50ffb6d0543cca4c1a2e5307b2c7b_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a2a50ffb6d0543cca4c1a2e5307b2c7b_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:36,573 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/6f2e8caaf87941278461d43a4ea6827a, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:36,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/6f2e8caaf87941278461d43a4ea6827a is 175, key is test_row_0/A:col10/1732052496515/Put/seqid=0 2024-11-19T21:41:36,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742205_1381 (size=56733) 2024-11-19T21:41:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T21:41:36,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T21:41:36,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:36,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:36,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052556658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052556668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052556668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052556668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052556668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T21:41:36,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:36,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:36,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T21:41:36,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052556868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052556878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052556878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052556878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052556878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:36,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T21:41:36,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:36,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:36,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:36,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:36,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/6f2e8caaf87941278461d43a4ea6827a 2024-11-19T21:41:37,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/6b2cf996aa844042a88dbc7e2c823fcd is 50, key is test_row_0/B:col10/1732052496515/Put/seqid=0 2024-11-19T21:41:37,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742206_1382 (size=12001) 2024-11-19T21:41:37,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T21:41:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:37,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T21:41:37,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052557178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052557198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052557198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052557198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052557198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T21:41:37,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:37,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:37,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:37,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:37,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:37,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/6b2cf996aa844042a88dbc7e2c823fcd 2024-11-19T21:41:37,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T21:41:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:37,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:37,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/0268c4cf6abf471bac2dcf4440212478 is 50, key is test_row_0/C:col10/1732052496515/Put/seqid=0 2024-11-19T21:41:37,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742207_1383 (size=12001) 2024-11-19T21:41:37,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/0268c4cf6abf471bac2dcf4440212478 2024-11-19T21:41:37,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/6f2e8caaf87941278461d43a4ea6827a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/6f2e8caaf87941278461d43a4ea6827a 2024-11-19T21:41:37,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/6f2e8caaf87941278461d43a4ea6827a, entries=300, sequenceid=17, filesize=55.4 K 2024-11-19T21:41:37,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/6b2cf996aa844042a88dbc7e2c823fcd as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/6b2cf996aa844042a88dbc7e2c823fcd 2024-11-19T21:41:37,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/6b2cf996aa844042a88dbc7e2c823fcd, entries=150, sequenceid=17, filesize=11.7 K 2024-11-19T21:41:37,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/0268c4cf6abf471bac2dcf4440212478 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0268c4cf6abf471bac2dcf4440212478 2024-11-19T21:41:37,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0268c4cf6abf471bac2dcf4440212478, entries=150, sequenceid=17, filesize=11.7 K 2024-11-19T21:41:37,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a87877009cc80e5fd95e37cd9db304a4 in 930ms, sequenceid=17, compaction requested=false 2024-11-19T21:41:37,445 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T21:41:37,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:37,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T21:41:37,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:37,569 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:41:37,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111918e6027cdacc46b9adeaff088cf64000_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052496550/Put/seqid=0 2024-11-19T21:41:37,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742208_1384 (size=12154) 2024-11-19T21:41:37,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:37,581 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111918e6027cdacc46b9adeaff088cf64000_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111918e6027cdacc46b9adeaff088cf64000_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/1f9a17e63eeb40fa827b1916c05f5be2, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:37,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/1f9a17e63eeb40fa827b1916c05f5be2 is 175, key is test_row_0/A:col10/1732052496550/Put/seqid=0 2024-11-19T21:41:37,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742209_1385 (size=30955) 2024-11-19T21:41:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T21:41:37,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:37,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052557717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052557718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052557721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052557721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052557722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052557828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052557828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052557838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052557838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:37,988 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/1f9a17e63eeb40fa827b1916c05f5be2 2024-11-19T21:41:37,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/a3dda099e96d44fa8a485d8d7c3857d6 is 50, key is test_row_0/B:col10/1732052496550/Put/seqid=0 2024-11-19T21:41:37,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742210_1386 (size=12001) 2024-11-19T21:41:37,999 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/a3dda099e96d44fa8a485d8d7c3857d6 2024-11-19T21:41:38,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/4c782d76b2d141e881945813aeb7728c is 50, key is test_row_0/C:col10/1732052496550/Put/seqid=0 2024-11-19T21:41:38,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742211_1387 (size=12001) 2024-11-19T21:41:38,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052558038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052558038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052558048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052558048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,123 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T21:41:38,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052558348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052558358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052558358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052558368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,419 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/4c782d76b2d141e881945813aeb7728c 2024-11-19T21:41:38,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/1f9a17e63eeb40fa827b1916c05f5be2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/1f9a17e63eeb40fa827b1916c05f5be2 2024-11-19T21:41:38,425 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/1f9a17e63eeb40fa827b1916c05f5be2, entries=150, sequenceid=40, filesize=30.2 K 2024-11-19T21:41:38,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/a3dda099e96d44fa8a485d8d7c3857d6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a3dda099e96d44fa8a485d8d7c3857d6 2024-11-19T21:41:38,428 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a3dda099e96d44fa8a485d8d7c3857d6, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T21:41:38,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/4c782d76b2d141e881945813aeb7728c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/4c782d76b2d141e881945813aeb7728c 2024-11-19T21:41:38,432 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/4c782d76b2d141e881945813aeb7728c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T21:41:38,432 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a87877009cc80e5fd95e37cd9db304a4 in 864ms, sequenceid=40, compaction requested=false 2024-11-19T21:41:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-19T21:41:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-19T21:41:38,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T21:41:38,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9380 sec 2024-11-19T21:41:38,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.9420 sec 2024-11-19T21:41:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T21:41:38,639 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:38,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-19T21:41:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T21:41:38,640 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:38,641 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:38,641 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:38,741 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:38,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T21:41:38,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b24fe985e89647e387c628e4b180a5da_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052497721/Put/seqid=0 2024-11-19T21:41:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T21:41:38,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742212_1388 (size=14594) 2024-11-19T21:41:38,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T21:41:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:38,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:38,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:38,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052558851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052558858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052558878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052558878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052558878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:38,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T21:41:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:38,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T21:41:38,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052558958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:39,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T21:41:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,152 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:39,155 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b24fe985e89647e387c628e4b180a5da_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b24fe985e89647e387c628e4b180a5da_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:39,156 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/f7b44d1833c6464682d0bafd2880f12a, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:39,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/f7b44d1833c6464682d0bafd2880f12a is 175, key is test_row_0/A:col10/1732052497721/Put/seqid=0 2024-11-19T21:41:39,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742213_1389 (size=39549) 2024-11-19T21:41:39,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052559167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:39,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T21:41:39,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T21:41:39,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T21:41:39,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:39,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052559478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:39,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T21:41:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,560 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/f7b44d1833c6464682d0bafd2880f12a 2024-11-19T21:41:39,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/d5a772313d96487783016f2b172c9c66 is 50, key is test_row_0/B:col10/1732052497721/Put/seqid=0 2024-11-19T21:41:39,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742214_1390 (size=12001) 2024-11-19T21:41:39,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T21:41:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T21:41:39,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T21:41:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:39,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052559868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:39,875 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:39,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052559898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:39,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052559898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052559898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:39,904 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,905 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:39,905 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:39,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/d5a772313d96487783016f2b172c9c66 2024-11-19T21:41:39,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/51bef94bdc0942498e16caab08ca4c00 is 50, key is test_row_0/C:col10/1732052497721/Put/seqid=0 2024-11-19T21:41:39,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742215_1391 (size=12001) 2024-11-19T21:41:39,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/51bef94bdc0942498e16caab08ca4c00 2024-11-19T21:41:39,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/f7b44d1833c6464682d0bafd2880f12a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f7b44d1833c6464682d0bafd2880f12a 2024-11-19T21:41:39,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f7b44d1833c6464682d0bafd2880f12a, entries=200, sequenceid=54, filesize=38.6 K 2024-11-19T21:41:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/d5a772313d96487783016f2b172c9c66 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d5a772313d96487783016f2b172c9c66 2024-11-19T21:41:39,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d5a772313d96487783016f2b172c9c66, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T21:41:39,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/51bef94bdc0942498e16caab08ca4c00 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/51bef94bdc0942498e16caab08ca4c00 2024-11-19T21:41:39,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/51bef94bdc0942498e16caab08ca4c00, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T21:41:39,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052559988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:39,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a87877009cc80e5fd95e37cd9db304a4 in 1255ms, sequenceid=54, compaction requested=true 2024-11-19T21:41:39,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:39,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:39,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:39,997 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:39,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:39,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:39,997 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:39,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:39,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:39,997 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:39,997 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:39,997 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/A is initiating minor compaction (all files) 2024-11-19T21:41:39,998 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/B is initiating minor compaction (all files) 2024-11-19T21:41:39,998 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/A in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,998 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/B in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,998 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/6f2e8caaf87941278461d43a4ea6827a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/1f9a17e63eeb40fa827b1916c05f5be2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f7b44d1833c6464682d0bafd2880f12a] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=124.3 K 2024-11-19T21:41:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:39,998 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:39,998 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/6f2e8caaf87941278461d43a4ea6827a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/1f9a17e63eeb40fa827b1916c05f5be2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f7b44d1833c6464682d0bafd2880f12a] 2024-11-19T21:41:39,998 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/6b2cf996aa844042a88dbc7e2c823fcd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a3dda099e96d44fa8a485d8d7c3857d6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d5a772313d96487783016f2b172c9c66] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=35.2 K 2024-11-19T21:41:39,998 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f2e8caaf87941278461d43a4ea6827a, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732052496506 2024-11-19T21:41:40,000 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b2cf996aa844042a88dbc7e2c823fcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732052496506 2024-11-19T21:41:40,000 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f9a17e63eeb40fa827b1916c05f5be2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732052496547 2024-11-19T21:41:40,000 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a3dda099e96d44fa8a485d8d7c3857d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732052496547 2024-11-19T21:41:40,000 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b44d1833c6464682d0bafd2880f12a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732052497716 2024-11-19T21:41:40,000 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a772313d96487783016f2b172c9c66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732052497716 2024-11-19T21:41:40,004 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:40,008 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411197c7b1012520c42e280d1be5ceaa029a1_a87877009cc80e5fd95e37cd9db304a4 store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:40,010 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:40,010 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/2b27e748675448d19660d4cd2c59a24f is 50, key is test_row_0/B:col10/1732052497721/Put/seqid=0 2024-11-19T21:41:40,011 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411197c7b1012520c42e280d1be5ceaa029a1_a87877009cc80e5fd95e37cd9db304a4, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:40,011 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197c7b1012520c42e280d1be5ceaa029a1_a87877009cc80e5fd95e37cd9db304a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:40,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T21:41:40,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742216_1392 (size=12104) 2024-11-19T21:41:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:40,015 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T21:41:40,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742217_1393 (size=4469) 2024-11-19T21:41:40,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a253e3749b8e485aa5bf85352b014218_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052498850/Put/seqid=0 2024-11-19T21:41:40,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742218_1394 (size=12154) 2024-11-19T21:41:40,420 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:40,421 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/d656ef1c60ca4d6f95311edaa3dc78aa is 175, key is test_row_0/A:col10/1732052497721/Put/seqid=0 2024-11-19T21:41:40,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:40,443 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a253e3749b8e485aa5bf85352b014218_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a253e3749b8e485aa5bf85352b014218_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:40,444 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/2b27e748675448d19660d4cd2c59a24f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/2b27e748675448d19660d4cd2c59a24f 2024-11-19T21:41:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/68a18cf481b8408a82efb2a2822c1a80, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:40,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/68a18cf481b8408a82efb2a2822c1a80 is 175, key is test_row_0/A:col10/1732052498850/Put/seqid=0 2024-11-19T21:41:40,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742219_1395 (size=31058) 2024-11-19T21:41:40,450 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/B of a87877009cc80e5fd95e37cd9db304a4 into 2b27e748675448d19660d4cd2c59a24f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:40,450 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:40,450 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/B, priority=13, startTime=1732052499997; duration=0sec 2024-11-19T21:41:40,451 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:40,451 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:B 2024-11-19T21:41:40,451 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:40,451 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:40,451 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/C is initiating minor compaction (all files) 2024-11-19T21:41:40,452 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/C in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:40,452 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0268c4cf6abf471bac2dcf4440212478, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/4c782d76b2d141e881945813aeb7728c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/51bef94bdc0942498e16caab08ca4c00] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=35.2 K 2024-11-19T21:41:40,452 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0268c4cf6abf471bac2dcf4440212478, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732052496506 2024-11-19T21:41:40,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742220_1396 (size=30955) 2024-11-19T21:41:40,452 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c782d76b2d141e881945813aeb7728c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732052496547 2024-11-19T21:41:40,452 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 51bef94bdc0942498e16caab08ca4c00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732052497716 2024-11-19T21:41:40,458 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:40,458 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/ff80eb089e7f405ab8ab77c9ff522fe5 is 50, key is test_row_0/C:col10/1732052497721/Put/seqid=0 2024-11-19T21:41:40,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742221_1397 (size=12104) 2024-11-19T21:41:40,464 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/ff80eb089e7f405ab8ab77c9ff522fe5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/ff80eb089e7f405ab8ab77c9ff522fe5 2024-11-19T21:41:40,468 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/C of a87877009cc80e5fd95e37cd9db304a4 into ff80eb089e7f405ab8ab77c9ff522fe5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:40,468 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:40,468 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/C, priority=13, startTime=1732052499997; duration=0sec 2024-11-19T21:41:40,468 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:40,468 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:C 2024-11-19T21:41:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T21:41:40,849 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/d656ef1c60ca4d6f95311edaa3dc78aa as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/d656ef1c60ca4d6f95311edaa3dc78aa 2024-11-19T21:41:40,851 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/68a18cf481b8408a82efb2a2822c1a80 2024-11-19T21:41:40,853 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/A of a87877009cc80e5fd95e37cd9db304a4 into d656ef1c60ca4d6f95311edaa3dc78aa(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:40,854 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:40,854 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/A, priority=13, startTime=1732052499996; duration=0sec 2024-11-19T21:41:40,854 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:40,854 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:A 2024-11-19T21:41:40,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/3b35ddd6c46e46c2b3a76f37d7739022 is 50, key is test_row_0/B:col10/1732052498850/Put/seqid=0 2024-11-19T21:41:40,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742222_1398 (size=12001) 2024-11-19T21:41:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:41,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:41,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052561067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:41,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052561178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:41,278 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/3b35ddd6c46e46c2b3a76f37d7739022 2024-11-19T21:41:41,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/64a8158310244dd9a7513dc83d735404 is 50, key is test_row_0/C:col10/1732052498850/Put/seqid=0 2024-11-19T21:41:41,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742223_1399 (size=12001) 2024-11-19T21:41:41,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052561388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:41,688 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/64a8158310244dd9a7513dc83d735404 2024-11-19T21:41:41,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/68a18cf481b8408a82efb2a2822c1a80 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/68a18cf481b8408a82efb2a2822c1a80 2024-11-19T21:41:41,694 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/68a18cf481b8408a82efb2a2822c1a80, entries=150, sequenceid=76, filesize=30.2 K 2024-11-19T21:41:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/3b35ddd6c46e46c2b3a76f37d7739022 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/3b35ddd6c46e46c2b3a76f37d7739022 2024-11-19T21:41:41,697 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/3b35ddd6c46e46c2b3a76f37d7739022, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T21:41:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T21:41:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:41,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/64a8158310244dd9a7513dc83d735404 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64a8158310244dd9a7513dc83d735404 2024-11-19T21:41:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052561698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:41,701 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64a8158310244dd9a7513dc83d735404, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T21:41:41,702 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a87877009cc80e5fd95e37cd9db304a4 in 1687ms, sequenceid=76, compaction requested=false 2024-11-19T21:41:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-19T21:41:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-19T21:41:41,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-19T21:41:41,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0620 sec 2024-11-19T21:41:41,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.0660 sec 2024-11-19T21:41:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:41,902 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:41,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T21:41:41,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ec0f2456132f4b9c86f678ddbf8d9659_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052501901/Put/seqid=0 2024-11-19T21:41:41,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742224_1400 (size=14594) 2024-11-19T21:41:41,915 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:41,918 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ec0f2456132f4b9c86f678ddbf8d9659_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ec0f2456132f4b9c86f678ddbf8d9659_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:41,919 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/597fea4fe6f746c4b220cb627ec11660, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:41,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/597fea4fe6f746c4b220cb627ec11660 is 175, key is test_row_0/A:col10/1732052501901/Put/seqid=0 2024-11-19T21:41:41,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742225_1401 (size=39549) 2024-11-19T21:41:41,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052561949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052561952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052561953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052561957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052562068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052562068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052562068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052562068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052562208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052562278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052562278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052562278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052562278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,323 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/597fea4fe6f746c4b220cb627ec11660 2024-11-19T21:41:42,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/cc7774015e2649649e4a6e7999e51e3c is 50, key is test_row_0/B:col10/1732052501901/Put/seqid=0 2024-11-19T21:41:42,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742226_1402 (size=12001) 2024-11-19T21:41:42,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052562588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052562589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052562589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052562589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:42,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/cc7774015e2649649e4a6e7999e51e3c 2024-11-19T21:41:42,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/1b79f3106fbc4327a3d4b7409798e6a8 is 50, key is test_row_0/C:col10/1732052501901/Put/seqid=0 2024-11-19T21:41:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742227_1403 (size=12001) 2024-11-19T21:41:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T21:41:42,809 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:42,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-19T21:41:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T21:41:42,811 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:42,811 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:42,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T21:41:42,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:42,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:42,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:42,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:42,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:42,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052563098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:43,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052563098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052563098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:43,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052563108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:43,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:43,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:43,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T21:41:43,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/1b79f3106fbc4327a3d4b7409798e6a8 2024-11-19T21:41:43,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/597fea4fe6f746c4b220cb627ec11660 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/597fea4fe6f746c4b220cb627ec11660 2024-11-19T21:41:43,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/597fea4fe6f746c4b220cb627ec11660, entries=200, sequenceid=94, filesize=38.6 K 2024-11-19T21:41:43,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/cc7774015e2649649e4a6e7999e51e3c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/cc7774015e2649649e4a6e7999e51e3c 2024-11-19T21:41:43,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/cc7774015e2649649e4a6e7999e51e3c, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T21:41:43,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/1b79f3106fbc4327a3d4b7409798e6a8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/1b79f3106fbc4327a3d4b7409798e6a8 2024-11-19T21:41:43,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/1b79f3106fbc4327a3d4b7409798e6a8, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T21:41:43,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a87877009cc80e5fd95e37cd9db304a4 in 1257ms, sequenceid=94, compaction requested=true 2024-11-19T21:41:43,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:43,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:43,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:43,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:43,159 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:43,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:43,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:43,159 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:43,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:43,160 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:43,160 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:43,160 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/A is initiating minor compaction (all files) 2024-11-19T21:41:43,160 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/B is initiating minor compaction (all files) 2024-11-19T21:41:43,160 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/A in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,160 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/B in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,160 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/d656ef1c60ca4d6f95311edaa3dc78aa, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/68a18cf481b8408a82efb2a2822c1a80, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/597fea4fe6f746c4b220cb627ec11660] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=99.2 K 2024-11-19T21:41:43,160 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/2b27e748675448d19660d4cd2c59a24f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/3b35ddd6c46e46c2b3a76f37d7739022, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/cc7774015e2649649e4a6e7999e51e3c] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=35.3 K 2024-11-19T21:41:43,160 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,160 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/d656ef1c60ca4d6f95311edaa3dc78aa, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/68a18cf481b8408a82efb2a2822c1a80, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/597fea4fe6f746c4b220cb627ec11660] 2024-11-19T21:41:43,160 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b27e748675448d19660d4cd2c59a24f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732052497716 2024-11-19T21:41:43,160 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting d656ef1c60ca4d6f95311edaa3dc78aa, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732052497716 2024-11-19T21:41:43,160 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b35ddd6c46e46c2b3a76f37d7739022, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732052498817 2024-11-19T21:41:43,160 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68a18cf481b8408a82efb2a2822c1a80, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732052498817 2024-11-19T21:41:43,161 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting cc7774015e2649649e4a6e7999e51e3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052501053 2024-11-19T21:41:43,161 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 597fea4fe6f746c4b220cb627ec11660, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052501053 2024-11-19T21:41:43,166 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:43,168 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:43,168 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/f23f367a395b4d93b953936261c7b8d8 is 50, key is test_row_0/B:col10/1732052501901/Put/seqid=0 2024-11-19T21:41:43,172 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411190418888cec574cc49f7f25a08efbb9dc_a87877009cc80e5fd95e37cd9db304a4 store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:43,174 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411190418888cec574cc49f7f25a08efbb9dc_a87877009cc80e5fd95e37cd9db304a4, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:43,174 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190418888cec574cc49f7f25a08efbb9dc_a87877009cc80e5fd95e37cd9db304a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:43,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742228_1404 (size=12207) 2024-11-19T21:41:43,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742229_1405 (size=4469) 2024-11-19T21:41:43,192 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#A#compaction#349 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:43,192 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/c1cf2bef643e418a8c5bb42d13741b71 is 175, key is test_row_0/A:col10/1732052501901/Put/seqid=0 2024-11-19T21:41:43,193 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/f23f367a395b4d93b953936261c7b8d8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/f23f367a395b4d93b953936261c7b8d8 2024-11-19T21:41:43,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742230_1406 (size=31161) 2024-11-19T21:41:43,197 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/B of a87877009cc80e5fd95e37cd9db304a4 into f23f367a395b4d93b953936261c7b8d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:43,197 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:43,197 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/B, priority=13, startTime=1732052503159; duration=0sec 2024-11-19T21:41:43,197 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:43,197 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:B 2024-11-19T21:41:43,198 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:43,198 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:43,199 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/C is initiating minor compaction (all files) 2024-11-19T21:41:43,199 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/C in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,199 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/ff80eb089e7f405ab8ab77c9ff522fe5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64a8158310244dd9a7513dc83d735404, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/1b79f3106fbc4327a3d4b7409798e6a8] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=35.3 K 2024-11-19T21:41:43,199 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting ff80eb089e7f405ab8ab77c9ff522fe5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732052497716 2024-11-19T21:41:43,199 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 64a8158310244dd9a7513dc83d735404, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732052498817 2024-11-19T21:41:43,199 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b79f3106fbc4327a3d4b7409798e6a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052501053 2024-11-19T21:41:43,203 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/c1cf2bef643e418a8c5bb42d13741b71 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c1cf2bef643e418a8c5bb42d13741b71 2024-11-19T21:41:43,207 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:43,208 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/489337d91efb4a7c93e731e3dc5519ac is 50, key is test_row_0/C:col10/1732052501901/Put/seqid=0 2024-11-19T21:41:43,209 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/A of a87877009cc80e5fd95e37cd9db304a4 into c1cf2bef643e418a8c5bb42d13741b71(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:43,209 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:43,209 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/A, priority=13, startTime=1732052503159; duration=0sec 2024-11-19T21:41:43,209 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:43,209 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:A 2024-11-19T21:41:43,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742231_1407 (size=12207) 2024-11-19T21:41:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:43,233 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:43,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T21:41:43,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119acfa745e4f534e438f47e04abd9a3872_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052503232/Put/seqid=0 2024-11-19T21:41:43,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742232_1408 (size=14594) 2024-11-19T21:41:43,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:43,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:43,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052563286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:43,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052563398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:43,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T21:41:43,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T21:41:43,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T21:41:43,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052563608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:43,616 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/489337d91efb4a7c93e731e3dc5519ac as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/489337d91efb4a7c93e731e3dc5519ac 2024-11-19T21:41:43,620 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/C of a87877009cc80e5fd95e37cd9db304a4 into 489337d91efb4a7c93e731e3dc5519ac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:43,620 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:43,620 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/C, priority=13, startTime=1732052503159; duration=0sec 2024-11-19T21:41:43,620 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:43,620 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:C 2024-11-19T21:41:43,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:43,647 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119acfa745e4f534e438f47e04abd9a3872_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119acfa745e4f534e438f47e04abd9a3872_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:43,648 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/020db67e573a41f79a11371e0a45fbae, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:43,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/020db67e573a41f79a11371e0a45fbae is 175, key is test_row_0/A:col10/1732052503232/Put/seqid=0 2024-11-19T21:41:43,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742233_1409 (size=39549) 2024-11-19T21:41:43,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:43,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:43,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:43,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052563918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T21:41:44,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:44,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:44,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/020db67e573a41f79a11371e0a45fbae 2024-11-19T21:41:44,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/d020fcfa99ea4ae0a858a7903bfe9414 is 50, key is test_row_0/B:col10/1732052503232/Put/seqid=0 2024-11-19T21:41:44,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742234_1410 (size=12001) 2024-11-19T21:41:44,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052564108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:44,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052564108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:44,115 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,115 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:44,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052564108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:44,116 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:44,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052564118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:44,124 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:44,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:44,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052564438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:44,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/d020fcfa99ea4ae0a858a7903bfe9414 2024-11-19T21:41:44,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/b18e2695595b496c9805a8b89ae7db8c is 50, key is test_row_0/C:col10/1732052503232/Put/seqid=0 2024-11-19T21:41:44,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742235_1411 (size=12001) 2024-11-19T21:41:44,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:44,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/b18e2695595b496c9805a8b89ae7db8c 2024-11-19T21:41:44,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/020db67e573a41f79a11371e0a45fbae as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/020db67e573a41f79a11371e0a45fbae 2024-11-19T21:41:44,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/020db67e573a41f79a11371e0a45fbae, entries=200, sequenceid=118, filesize=38.6 K 2024-11-19T21:41:44,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/d020fcfa99ea4ae0a858a7903bfe9414 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d020fcfa99ea4ae0a858a7903bfe9414 2024-11-19T21:41:44,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d020fcfa99ea4ae0a858a7903bfe9414, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T21:41:44,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/b18e2695595b496c9805a8b89ae7db8c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b18e2695595b496c9805a8b89ae7db8c 2024-11-19T21:41:44,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b18e2695595b496c9805a8b89ae7db8c, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T21:41:44,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a87877009cc80e5fd95e37cd9db304a4 in 1650ms, sequenceid=118, compaction requested=false 2024-11-19T21:41:44,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:44,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T21:41:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:44,948 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T21:41:44,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d892a00c289845858ddfd581ac83f667_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052503252/Put/seqid=0 2024-11-19T21:41:44,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742236_1412 (size=12254) 2024-11-19T21:41:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T21:41:45,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:45,361 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d892a00c289845858ddfd581ac83f667_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d892a00c289845858ddfd581ac83f667_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:45,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/edec103fb2d24437bb52889f0d3aa584, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:45,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/edec103fb2d24437bb52889f0d3aa584 is 175, key is test_row_0/A:col10/1732052503252/Put/seqid=0 2024-11-19T21:41:45,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742237_1413 (size=31055) 2024-11-19T21:41:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:45,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052565549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:45,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052565658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:45,766 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/edec103fb2d24437bb52889f0d3aa584 2024-11-19T21:41:45,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/43139c79d1c24fa2a554fac74fc1d952 is 50, key is test_row_0/B:col10/1732052503252/Put/seqid=0 2024-11-19T21:41:45,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742238_1414 (size=12101) 2024-11-19T21:41:45,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052565868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:46,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052566128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:46,133 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:46,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052566138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:46,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052566138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:46,142 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:46,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:46,143 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:46,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:46,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052566148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:46,152 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:46,175 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/43139c79d1c24fa2a554fac74fc1d952 2024-11-19T21:41:46,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/a81f8a9b21f24500adbb23eb1099807f is 50, key is test_row_0/C:col10/1732052503252/Put/seqid=0 2024-11-19T21:41:46,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052566178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:46,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742239_1415 (size=12101) 2024-11-19T21:41:46,585 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/a81f8a9b21f24500adbb23eb1099807f 2024-11-19T21:41:46,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/edec103fb2d24437bb52889f0d3aa584 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/edec103fb2d24437bb52889f0d3aa584 2024-11-19T21:41:46,593 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/edec103fb2d24437bb52889f0d3aa584, entries=150, sequenceid=133, filesize=30.3 K 2024-11-19T21:41:46,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/43139c79d1c24fa2a554fac74fc1d952 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/43139c79d1c24fa2a554fac74fc1d952 2024-11-19T21:41:46,597 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/43139c79d1c24fa2a554fac74fc1d952, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T21:41:46,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/a81f8a9b21f24500adbb23eb1099807f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/a81f8a9b21f24500adbb23eb1099807f 2024-11-19T21:41:46,602 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/a81f8a9b21f24500adbb23eb1099807f, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T21:41:46,602 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a87877009cc80e5fd95e37cd9db304a4 in 1654ms, sequenceid=133, compaction requested=true 2024-11-19T21:41:46,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:46,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:46,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-19T21:41:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-19T21:41:46,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T21:41:46,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7930 sec 2024-11-19T21:41:46,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.7970 sec 2024-11-19T21:41:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:46,692 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:46,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:41:46,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111942ac82b4efd44dc8b44b9ff0e8349024_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052506691/Put/seqid=0 2024-11-19T21:41:46,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742240_1416 (size=14794) 2024-11-19T21:41:46,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052566751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:46,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052566868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T21:41:46,978 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:46,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-19T21:41:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T21:41:46,980 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:46,980 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:46,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:47,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052567078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T21:41:47,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:47,106 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111942ac82b4efd44dc8b44b9ff0e8349024_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111942ac82b4efd44dc8b44b9ff0e8349024_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:47,107 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/4f0e26c7d771470a8569724c79163679, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:47,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/4f0e26c7d771470a8569724c79163679 is 175, key is test_row_0/A:col10/1732052506691/Put/seqid=0 2024-11-19T21:41:47,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742241_1417 (size=39749) 2024-11-19T21:41:47,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T21:41:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T21:41:47,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:47,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T21:41:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052567398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:47,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T21:41:47,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:47,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/4f0e26c7d771470a8569724c79163679 2024-11-19T21:41:47,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c4020ec36bfa4321b10beb687fdd2d7e is 50, key is test_row_0/B:col10/1732052506691/Put/seqid=0 2024-11-19T21:41:47,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742242_1418 (size=12151) 2024-11-19T21:41:47,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T21:41:47,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:47,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T21:41:47,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T21:41:47,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:47,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T21:41:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:47,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c4020ec36bfa4321b10beb687fdd2d7e 2024-11-19T21:41:47,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052567918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:47,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/0e012f5056b344488e144f75b71e04af is 50, key is test_row_0/C:col10/1732052506691/Put/seqid=0 2024-11-19T21:41:47,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742243_1419 (size=12151) 2024-11-19T21:41:47,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/0e012f5056b344488e144f75b71e04af 2024-11-19T21:41:47,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/4f0e26c7d771470a8569724c79163679 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/4f0e26c7d771470a8569724c79163679 2024-11-19T21:41:47,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/4f0e26c7d771470a8569724c79163679, entries=200, sequenceid=156, filesize=38.8 K 2024-11-19T21:41:47,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c4020ec36bfa4321b10beb687fdd2d7e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c4020ec36bfa4321b10beb687fdd2d7e 2024-11-19T21:41:47,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c4020ec36bfa4321b10beb687fdd2d7e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T21:41:47,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/0e012f5056b344488e144f75b71e04af as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0e012f5056b344488e144f75b71e04af 2024-11-19T21:41:47,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0e012f5056b344488e144f75b71e04af, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T21:41:47,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a87877009cc80e5fd95e37cd9db304a4 in 1253ms, sequenceid=156, compaction requested=true 2024-11-19T21:41:47,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:47,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:47,945 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:41:47,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:47,945 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:41:47,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:47,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:47,947 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:41:47,947 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:41:47,947 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/A is initiating minor compaction (all files) 2024-11-19T21:41:47,947 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/B is initiating minor compaction (all files) 2024-11-19T21:41:47,947 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/B in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,947 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/A in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,947 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c1cf2bef643e418a8c5bb42d13741b71, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/020db67e573a41f79a11371e0a45fbae, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/edec103fb2d24437bb52889f0d3aa584, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/4f0e26c7d771470a8569724c79163679] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=138.2 K 2024-11-19T21:41:47,947 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/f23f367a395b4d93b953936261c7b8d8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d020fcfa99ea4ae0a858a7903bfe9414, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/43139c79d1c24fa2a554fac74fc1d952, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c4020ec36bfa4321b10beb687fdd2d7e] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=47.3 K 2024-11-19T21:41:47,947 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:47,947 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c1cf2bef643e418a8c5bb42d13741b71, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/020db67e573a41f79a11371e0a45fbae, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/edec103fb2d24437bb52889f0d3aa584, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/4f0e26c7d771470a8569724c79163679] 2024-11-19T21:41:47,947 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting f23f367a395b4d93b953936261c7b8d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052501053 2024-11-19T21:41:47,947 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1cf2bef643e418a8c5bb42d13741b71, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052501053 2024-11-19T21:41:47,947 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d020fcfa99ea4ae0a858a7903bfe9414, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732052501936 2024-11-19T21:41:47,947 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 020db67e573a41f79a11371e0a45fbae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732052501936 2024-11-19T21:41:47,948 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 43139c79d1c24fa2a554fac74fc1d952, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732052503252 2024-11-19T21:41:47,948 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting edec103fb2d24437bb52889f0d3aa584, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732052503252 2024-11-19T21:41:47,948 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f0e26c7d771470a8569724c79163679, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732052505513 2024-11-19T21:41:47,948 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c4020ec36bfa4321b10beb687fdd2d7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732052505538 2024-11-19T21:41:47,952 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:47,954 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#B#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:47,955 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/620e299daed5418696ad34f49d265fef is 50, key is test_row_0/B:col10/1732052506691/Put/seqid=0 2024-11-19T21:41:47,955 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119c3eb426c0bc6460d84f8a2b2f5968bc3_a87877009cc80e5fd95e37cd9db304a4 store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:47,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742244_1420 (size=12493) 2024-11-19T21:41:47,960 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119c3eb426c0bc6460d84f8a2b2f5968bc3_a87877009cc80e5fd95e37cd9db304a4, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:47,960 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c3eb426c0bc6460d84f8a2b2f5968bc3_a87877009cc80e5fd95e37cd9db304a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:47,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742245_1421 (size=4469) 2024-11-19T21:41:48,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T21:41:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:48,049 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:41:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194cb58af773c4492b8147037e193d74c5_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052506732/Put/seqid=0 2024-11-19T21:41:48,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742246_1422 (size=12304) 2024-11-19T21:41:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T21:41:48,361 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/620e299daed5418696ad34f49d265fef as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/620e299daed5418696ad34f49d265fef 2024-11-19T21:41:48,364 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#A#compaction#361 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:48,364 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/adac64c05a0141b6bd65e4599e9e1f28 is 175, key is test_row_0/A:col10/1732052506691/Put/seqid=0 2024-11-19T21:41:48,365 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/B of a87877009cc80e5fd95e37cd9db304a4 into 620e299daed5418696ad34f49d265fef(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:48,365 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:48,365 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/B, priority=12, startTime=1732052507945; duration=0sec 2024-11-19T21:41:48,365 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:48,365 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:B 2024-11-19T21:41:48,365 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T21:41:48,366 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T21:41:48,366 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/C is initiating minor compaction (all files) 2024-11-19T21:41:48,366 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/C in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:48,366 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/489337d91efb4a7c93e731e3dc5519ac, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b18e2695595b496c9805a8b89ae7db8c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/a81f8a9b21f24500adbb23eb1099807f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0e012f5056b344488e144f75b71e04af] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=47.3 K 2024-11-19T21:41:48,366 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 489337d91efb4a7c93e731e3dc5519ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732052501053 2024-11-19T21:41:48,367 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b18e2695595b496c9805a8b89ae7db8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732052501936 2024-11-19T21:41:48,367 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a81f8a9b21f24500adbb23eb1099807f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732052503252 2024-11-19T21:41:48,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742247_1423 (size=31447) 2024-11-19T21:41:48,367 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e012f5056b344488e144f75b71e04af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732052505538 2024-11-19T21:41:48,374 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:48,374 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/7228f159eedf4d4390ca079f46edca8a is 50, key is test_row_0/C:col10/1732052506691/Put/seqid=0 2024-11-19T21:41:48,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742248_1424 (size=12493) 2024-11-19T21:41:48,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,469 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194cb58af773c4492b8147037e193d74c5_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194cb58af773c4492b8147037e193d74c5_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:48,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/bc26f2b202fd4108bdee113dbcefe61e, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:48,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/bc26f2b202fd4108bdee113dbcefe61e is 175, key is test_row_0/A:col10/1732052506732/Put/seqid=0 2024-11-19T21:41:48,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742249_1425 (size=31105) 2024-11-19T21:41:48,771 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/adac64c05a0141b6bd65e4599e9e1f28 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adac64c05a0141b6bd65e4599e9e1f28 2024-11-19T21:41:48,775 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/A of a87877009cc80e5fd95e37cd9db304a4 into adac64c05a0141b6bd65e4599e9e1f28(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:48,775 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:48,775 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/A, priority=12, startTime=1732052507945; duration=0sec 2024-11-19T21:41:48,775 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:48,775 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:A 2024-11-19T21:41:48,782 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/7228f159eedf4d4390ca079f46edca8a as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/7228f159eedf4d4390ca079f46edca8a 2024-11-19T21:41:48,786 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/C of a87877009cc80e5fd95e37cd9db304a4 into 7228f159eedf4d4390ca079f46edca8a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:48,786 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:48,786 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/C, priority=12, startTime=1732052507946; duration=0sec 2024-11-19T21:41:48,786 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:48,786 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:C 2024-11-19T21:41:48,873 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/bc26f2b202fd4108bdee113dbcefe61e 2024-11-19T21:41:48,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/bcc70fd412254bdaa48df24f1233b694 is 50, key is test_row_0/B:col10/1732052506732/Put/seqid=0 2024-11-19T21:41:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742250_1426 (size=12151) 2024-11-19T21:41:48,883 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/bcc70fd412254bdaa48df24f1233b694 2024-11-19T21:41:48,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/9083c3ca2c6240a4952116b3741a9a98 is 50, key is test_row_0/C:col10/1732052506732/Put/seqid=0 2024-11-19T21:41:48,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742251_1427 (size=12151) 2024-11-19T21:41:48,892 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/9083c3ca2c6240a4952116b3741a9a98 2024-11-19T21:41:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/bc26f2b202fd4108bdee113dbcefe61e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/bc26f2b202fd4108bdee113dbcefe61e 2024-11-19T21:41:48,901 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/bc26f2b202fd4108bdee113dbcefe61e, entries=150, sequenceid=169, filesize=30.4 K 2024-11-19T21:41:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/bcc70fd412254bdaa48df24f1233b694 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/bcc70fd412254bdaa48df24f1233b694 2024-11-19T21:41:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,905 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/bcc70fd412254bdaa48df24f1233b694, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T21:41:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/9083c3ca2c6240a4952116b3741a9a98 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/9083c3ca2c6240a4952116b3741a9a98 2024-11-19T21:41:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,909 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/9083c3ca2c6240a4952116b3741a9a98, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T21:41:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,910 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for a87877009cc80e5fd95e37cd9db304a4 in 861ms, sequenceid=169, compaction requested=false 2024-11-19T21:41:48,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:48,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:48,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-19T21:41:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-19T21:41:48,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T21:41:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9300 sec 2024-11-19T21:41:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.9350 sec 2024-11-19T21:41:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:49,107 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:49,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:41:49,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119108bacf3519241d78fca794256cbd4bb_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052509094/Put/seqid=0 2024-11-19T21:41:49,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742252_1428 (size=14794) 2024-11-19T21:41:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T21:41:49,128 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:49,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T21:41:49,131 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T21:41:49,131 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:49,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T21:41:49,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052569316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:49,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:49,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052569428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T21:41:49,528 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:49,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119108bacf3519241d78fca794256cbd4bb_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119108bacf3519241d78fca794256cbd4bb_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:49,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/adf42a9f954f4d24a021886f54e02101, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:49,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/adf42a9f954f4d24a021886f54e02101 is 175, key is test_row_0/A:col10/1732052509094/Put/seqid=0 2024-11-19T21:41:49,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742253_1429 (size=39749) 2024-11-19T21:41:49,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052569648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:49,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:49,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T21:41:49,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:49,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:49,937 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/adf42a9f954f4d24a021886f54e02101 2024-11-19T21:41:49,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/a227b02e19094b539c11ac0a55562ff6 is 50, key is test_row_0/B:col10/1732052509094/Put/seqid=0 2024-11-19T21:41:49,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742254_1430 (size=12151) 2024-11-19T21:41:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052569958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:50,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38996 deadline: 1732052570148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38950 deadline: 1732052570148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:50,154 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:50,154 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:50,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38968 deadline: 1732052570178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:50,183 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:50,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39030 deadline: 1732052570197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:50,205 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in a87877009cc80e5fd95e37cd9db304a4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:41:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:50,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T21:41:50,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/a227b02e19094b539c11ac0a55562ff6 2024-11-19T21:41:50,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/c48dbb36c23f4adb9ca19fad4ff4e80f is 50, key is test_row_0/C:col10/1732052509094/Put/seqid=0 2024-11-19T21:41:50,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742255_1431 (size=12151) 2024-11-19T21:41:50,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052570468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:50,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:50,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:50,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/c48dbb36c23f4adb9ca19fad4ff4e80f 2024-11-19T21:41:50,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/adf42a9f954f4d24a021886f54e02101 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adf42a9f954f4d24a021886f54e02101 2024-11-19T21:41:50,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adf42a9f954f4d24a021886f54e02101, entries=200, sequenceid=183, filesize=38.8 K 2024-11-19T21:41:50,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/a227b02e19094b539c11ac0a55562ff6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a227b02e19094b539c11ac0a55562ff6 2024-11-19T21:41:50,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a227b02e19094b539c11ac0a55562ff6, entries=150, sequenceid=183, filesize=11.9 K 2024-11-19T21:41:50,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/c48dbb36c23f4adb9ca19fad4ff4e80f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/c48dbb36c23f4adb9ca19fad4ff4e80f 2024-11-19T21:41:50,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/c48dbb36c23f4adb9ca19fad4ff4e80f, entries=150, sequenceid=183, filesize=11.9 K 2024-11-19T21:41:50,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a87877009cc80e5fd95e37cd9db304a4 in 1681ms, sequenceid=183, compaction requested=true 2024-11-19T21:41:50,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:50,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:50,788 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:50,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:50,788 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:50,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:50,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:50,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:50,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:50,789 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:50,789 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:50,789 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/A is initiating minor compaction (all files) 2024-11-19T21:41:50,789 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/B is initiating minor compaction (all files) 2024-11-19T21:41:50,789 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/B in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,789 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/A in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,789 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adac64c05a0141b6bd65e4599e9e1f28, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/bc26f2b202fd4108bdee113dbcefe61e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adf42a9f954f4d24a021886f54e02101] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=99.9 K 2024-11-19T21:41:50,789 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/620e299daed5418696ad34f49d265fef, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/bcc70fd412254bdaa48df24f1233b694, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a227b02e19094b539c11ac0a55562ff6] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=35.9 K 2024-11-19T21:41:50,789 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,789 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adac64c05a0141b6bd65e4599e9e1f28, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/bc26f2b202fd4108bdee113dbcefe61e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adf42a9f954f4d24a021886f54e02101] 2024-11-19T21:41:50,789 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting adac64c05a0141b6bd65e4599e9e1f28, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732052505538 2024-11-19T21:41:50,789 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 620e299daed5418696ad34f49d265fef, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732052505538 2024-11-19T21:41:50,790 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc26f2b202fd4108bdee113dbcefe61e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732052506732 2024-11-19T21:41:50,790 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting bcc70fd412254bdaa48df24f1233b694, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732052506732 2024-11-19T21:41:50,790 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting adf42a9f954f4d24a021886f54e02101, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732052509085 2024-11-19T21:41:50,792 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a227b02e19094b539c11ac0a55562ff6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732052509094 2024-11-19T21:41:50,794 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:50,796 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198b3a9008960046469713d5336f3c6f37_a87877009cc80e5fd95e37cd9db304a4 store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:50,797 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#B#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:50,797 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198b3a9008960046469713d5336f3c6f37_a87877009cc80e5fd95e37cd9db304a4, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:50,797 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198b3a9008960046469713d5336f3c6f37_a87877009cc80e5fd95e37cd9db304a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:50,798 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c7e8b11de4c14599a5606e51ae4e1089 is 50, key is test_row_0/B:col10/1732052509094/Put/seqid=0 2024-11-19T21:41:50,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742256_1432 (size=12595) 2024-11-19T21:41:50,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742257_1433 (size=4469) 2024-11-19T21:41:50,809 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#A#compaction#369 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:50,809 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/87c89f2d9b994bccabf7d346d10b17ed is 175, key is test_row_0/A:col10/1732052509094/Put/seqid=0 2024-11-19T21:41:50,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T21:41:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,811 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:41:50,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742258_1434 (size=31549) 2024-11-19T21:41:50,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199f8d5afb96d644a5a51c138465326419_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052509293/Put/seqid=0 2024-11-19T21:41:50,820 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/87c89f2d9b994bccabf7d346d10b17ed as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/87c89f2d9b994bccabf7d346d10b17ed 2024-11-19T21:41:50,825 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/A of a87877009cc80e5fd95e37cd9db304a4 into 87c89f2d9b994bccabf7d346d10b17ed(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:50,825 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:50,825 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/A, priority=13, startTime=1732052510788; duration=0sec 2024-11-19T21:41:50,825 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:50,825 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:A 2024-11-19T21:41:50,825 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:50,827 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:50,827 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/C is initiating minor compaction (all files) 2024-11-19T21:41:50,827 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/C in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:50,827 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/7228f159eedf4d4390ca079f46edca8a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/9083c3ca2c6240a4952116b3741a9a98, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/c48dbb36c23f4adb9ca19fad4ff4e80f] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=35.9 K 2024-11-19T21:41:50,827 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7228f159eedf4d4390ca079f46edca8a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732052505538 2024-11-19T21:41:50,828 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9083c3ca2c6240a4952116b3741a9a98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732052506732 2024-11-19T21:41:50,828 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting c48dbb36c23f4adb9ca19fad4ff4e80f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732052509094 2024-11-19T21:41:50,834 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:50,835 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/e6ebb711b649424798e2f058bb61c998 is 50, key is test_row_0/C:col10/1732052509094/Put/seqid=0 2024-11-19T21:41:50,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742259_1435 (size=12304) 2024-11-19T21:41:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:50,852 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199f8d5afb96d644a5a51c138465326419_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199f8d5afb96d644a5a51c138465326419_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:50,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/81060eea71cb404f90774774e6656224, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:50,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/81060eea71cb404f90774774e6656224 is 175, key is test_row_0/A:col10/1732052509293/Put/seqid=0 2024-11-19T21:41:50,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742260_1436 (size=12595) 2024-11-19T21:41:50,862 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/e6ebb711b649424798e2f058bb61c998 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/e6ebb711b649424798e2f058bb61c998 2024-11-19T21:41:50,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742261_1437 (size=31105) 2024-11-19T21:41:50,869 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/C of a87877009cc80e5fd95e37cd9db304a4 into e6ebb711b649424798e2f058bb61c998(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:50,869 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:50,869 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/C, priority=13, startTime=1732052510788; duration=0sec 2024-11-19T21:41:50,869 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:50,869 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:C 2024-11-19T21:41:51,212 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c7e8b11de4c14599a5606e51ae4e1089 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c7e8b11de4c14599a5606e51ae4e1089 2024-11-19T21:41:51,216 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/B of a87877009cc80e5fd95e37cd9db304a4 into c7e8b11de4c14599a5606e51ae4e1089(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:51,216 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:51,216 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/B, priority=13, startTime=1732052510788; duration=0sec 2024-11-19T21:41:51,216 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:51,216 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:B 2024-11-19T21:41:51,269 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/81060eea71cb404f90774774e6656224 2024-11-19T21:41:51,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/50846bbc379b4a5f8986ba757a89d2c3 is 50, key is test_row_0/B:col10/1732052509293/Put/seqid=0 2024-11-19T21:41:51,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742262_1438 (size=12151) 2024-11-19T21:41:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T21:41:51,278 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/50846bbc379b4a5f8986ba757a89d2c3 2024-11-19T21:41:51,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/dd0648ef34a445879c4f26a7bad839ea is 50, key is test_row_0/C:col10/1732052509293/Put/seqid=0 2024-11-19T21:41:51,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742263_1439 (size=12151) 2024-11-19T21:41:51,288 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/dd0648ef34a445879c4f26a7bad839ea 2024-11-19T21:41:51,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/81060eea71cb404f90774774e6656224 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/81060eea71cb404f90774774e6656224 2024-11-19T21:41:51,295 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/81060eea71cb404f90774774e6656224, entries=150, sequenceid=208, filesize=30.4 K 2024-11-19T21:41:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/50846bbc379b4a5f8986ba757a89d2c3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/50846bbc379b4a5f8986ba757a89d2c3 2024-11-19T21:41:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,299 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/50846bbc379b4a5f8986ba757a89d2c3, entries=150, sequenceid=208, filesize=11.9 K 2024-11-19T21:41:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/dd0648ef34a445879c4f26a7bad839ea as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/dd0648ef34a445879c4f26a7bad839ea 2024-11-19T21:41:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,303 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/dd0648ef34a445879c4f26a7bad839ea, entries=150, sequenceid=208, filesize=11.9 K 2024-11-19T21:41:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,304 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for a87877009cc80e5fd95e37cd9db304a4 in 493ms, sequenceid=208, compaction requested=false 2024-11-19T21:41:51,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:51,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:51,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T21:41:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T21:41:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T21:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1740 sec 2024-11-19T21:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1780 sec 2024-11-19T21:41:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:51,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:41:51,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119045b3fd3629e4a32a5c1ad76d9d0c345_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052511640/Put/seqid=0 2024-11-19T21:41:51,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742264_1440 (size=22268) 2024-11-19T21:41:51,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052571812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:51,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:51,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:51,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052571928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:52,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:52,088 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119045b3fd3629e4a32a5c1ad76d9d0c345_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119045b3fd3629e4a32a5c1ad76d9d0c345_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:52,089 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/c517e8b2974e4fb8a92805a239e94100, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:52,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/c517e8b2974e4fb8a92805a239e94100 is 175, key is test_row_0/A:col10/1732052511640/Put/seqid=0 2024-11-19T21:41:52,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742265_1441 (size=65673) 2024-11-19T21:41:52,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052572138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:52,277 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T21:41:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052572447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:52,492 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/c517e8b2974e4fb8a92805a239e94100 2024-11-19T21:41:52,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/dc6dec65d5a94df2a827c473449e3d9f is 50, key is test_row_0/B:col10/1732052511640/Put/seqid=0 2024-11-19T21:41:52,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742266_1442 (size=12151) 2024-11-19T21:41:52,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/dc6dec65d5a94df2a827c473449e3d9f 2024-11-19T21:41:52,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/fcb83d33cc0b423bafd096ea82931c52 is 50, key is test_row_0/C:col10/1732052511640/Put/seqid=0 2024-11-19T21:41:52,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742267_1443 (size=12151) 2024-11-19T21:41:52,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052572968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T21:41:53,298 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:53,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-19T21:41:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T21:41:53,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:53,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:53,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:53,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/fcb83d33cc0b423bafd096ea82931c52 2024-11-19T21:41:53,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/c517e8b2974e4fb8a92805a239e94100 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c517e8b2974e4fb8a92805a239e94100 2024-11-19T21:41:53,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c517e8b2974e4fb8a92805a239e94100, entries=350, sequenceid=222, filesize=64.1 K 2024-11-19T21:41:53,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/dc6dec65d5a94df2a827c473449e3d9f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/dc6dec65d5a94df2a827c473449e3d9f 2024-11-19T21:41:53,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/dc6dec65d5a94df2a827c473449e3d9f, entries=150, sequenceid=222, filesize=11.9 K 2024-11-19T21:41:53,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/fcb83d33cc0b423bafd096ea82931c52 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/fcb83d33cc0b423bafd096ea82931c52 2024-11-19T21:41:53,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/fcb83d33cc0b423bafd096ea82931c52, entries=150, sequenceid=222, filesize=11.9 K 2024-11-19T21:41:53,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a87877009cc80e5fd95e37cd9db304a4 in 1668ms, sequenceid=222, compaction requested=true 2024-11-19T21:41:53,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:53,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:53,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:53,330 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:53,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:53,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:53,330 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:53,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:53,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:53,331 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128327 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:53,331 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:53,331 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/A is initiating minor compaction (all files) 2024-11-19T21:41:53,331 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/B is initiating minor compaction (all files) 2024-11-19T21:41:53,331 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/A in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:53,331 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/B in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:53,331 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/87c89f2d9b994bccabf7d346d10b17ed, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/81060eea71cb404f90774774e6656224, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c517e8b2974e4fb8a92805a239e94100] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=125.3 K 2024-11-19T21:41:53,331 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c7e8b11de4c14599a5606e51ae4e1089, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/50846bbc379b4a5f8986ba757a89d2c3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/dc6dec65d5a94df2a827c473449e3d9f] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=36.0 K 2024-11-19T21:41:53,331 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:53,331 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/87c89f2d9b994bccabf7d346d10b17ed, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/81060eea71cb404f90774774e6656224, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c517e8b2974e4fb8a92805a239e94100] 2024-11-19T21:41:53,332 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c89f2d9b994bccabf7d346d10b17ed, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732052509094 2024-11-19T21:41:53,332 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c7e8b11de4c14599a5606e51ae4e1089, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732052509094 2024-11-19T21:41:53,332 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81060eea71cb404f90774774e6656224, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732052509230 2024-11-19T21:41:53,332 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 50846bbc379b4a5f8986ba757a89d2c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732052509230 2024-11-19T21:41:53,332 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting dc6dec65d5a94df2a827c473449e3d9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732052511640 2024-11-19T21:41:53,332 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting c517e8b2974e4fb8a92805a239e94100, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732052511523 2024-11-19T21:41:53,339 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:53,339 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/39334c147897408b9298a150150fbb49 is 50, key is test_row_0/B:col10/1732052511640/Put/seqid=0 2024-11-19T21:41:53,340 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:53,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742268_1444 (size=12697) 2024-11-19T21:41:53,347 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111920ec8a870df247f1a1f155db56d6f969_a87877009cc80e5fd95e37cd9db304a4 store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:53,349 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111920ec8a870df247f1a1f155db56d6f969_a87877009cc80e5fd95e37cd9db304a4, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:53,350 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111920ec8a870df247f1a1f155db56d6f969_a87877009cc80e5fd95e37cd9db304a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:53,358 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/39334c147897408b9298a150150fbb49 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/39334c147897408b9298a150150fbb49 2024-11-19T21:41:53,361 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/B of a87877009cc80e5fd95e37cd9db304a4 into 39334c147897408b9298a150150fbb49(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:53,361 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:53,361 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/B, priority=13, startTime=1732052513330; duration=0sec 2024-11-19T21:41:53,361 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:53,361 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:B 2024-11-19T21:41:53,361 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:53,362 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:53,362 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/C is initiating minor compaction (all files) 2024-11-19T21:41:53,362 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/C in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:53,362 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/e6ebb711b649424798e2f058bb61c998, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/dd0648ef34a445879c4f26a7bad839ea, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/fcb83d33cc0b423bafd096ea82931c52] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=36.0 K 2024-11-19T21:41:53,362 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ebb711b649424798e2f058bb61c998, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732052509094 2024-11-19T21:41:53,363 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting dd0648ef34a445879c4f26a7bad839ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732052509230 2024-11-19T21:41:53,363 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb83d33cc0b423bafd096ea82931c52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732052511640 2024-11-19T21:41:53,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742269_1445 (size=4469) 2024-11-19T21:41:53,369 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#A#compaction#379 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:53,370 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/75c09db68ab7448bad7f054a98477426 is 175, key is test_row_0/A:col10/1732052511640/Put/seqid=0 2024-11-19T21:41:53,379 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:53,380 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/6a235b1131294ad687298b79bd29a505 is 50, key is test_row_0/C:col10/1732052511640/Put/seqid=0 2024-11-19T21:41:53,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742270_1446 (size=31651) 2024-11-19T21:41:53,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742271_1447 (size=12697) 2024-11-19T21:41:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T21:41:53,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T21:41:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:53,453 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:41:53,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196be03743656b4d3ebe352590acddd00e_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052511811/Put/seqid=0 2024-11-19T21:41:53,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742272_1448 (size=12304) 2024-11-19T21:41:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T21:41:53,792 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/75c09db68ab7448bad7f054a98477426 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/75c09db68ab7448bad7f054a98477426 2024-11-19T21:41:53,796 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/A of a87877009cc80e5fd95e37cd9db304a4 into 75c09db68ab7448bad7f054a98477426(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:53,796 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:53,796 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/A, priority=13, startTime=1732052513330; duration=0sec 2024-11-19T21:41:53,796 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:53,796 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:A 2024-11-19T21:41:53,799 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/6a235b1131294ad687298b79bd29a505 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/6a235b1131294ad687298b79bd29a505 2024-11-19T21:41:53,803 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/C of a87877009cc80e5fd95e37cd9db304a4 into 6a235b1131294ad687298b79bd29a505(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:53,803 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:53,803 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/C, priority=13, startTime=1732052513330; duration=0sec 2024-11-19T21:41:53,803 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:53,803 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:C 2024-11-19T21:41:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:53,864 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196be03743656b4d3ebe352590acddd00e_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196be03743656b4d3ebe352590acddd00e_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:53,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/dba294da731146e8b874497334c0fc5d, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/dba294da731146e8b874497334c0fc5d is 175, key is test_row_0/A:col10/1732052511811/Put/seqid=0 2024-11-19T21:41:53,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742273_1449 (size=31105) 2024-11-19T21:41:53,869 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/dba294da731146e8b874497334c0fc5d 2024-11-19T21:41:53,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c958115d4d0643e1bd1d4af64264f150 is 50, key is test_row_0/B:col10/1732052511811/Put/seqid=0 2024-11-19T21:41:53,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742274_1450 (size=12151) 2024-11-19T21:41:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T21:41:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:53,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:54,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052574028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:54,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:54,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:54,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:54,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052574138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:54,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:54,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:54,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:54,277 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c958115d4d0643e1bd1d4af64264f150 2024-11-19T21:41:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/abf19c5557ca470e9b7dc2c433230d27 is 50, key is test_row_0/C:col10/1732052511811/Put/seqid=0 2024-11-19T21:41:54,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742275_1451 (size=12151) 2024-11-19T21:41:54,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052574358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T21:41:54,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052574668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:54,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:54,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:54,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:54,687 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/abf19c5557ca470e9b7dc2c433230d27 2024-11-19T21:41:54,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/dba294da731146e8b874497334c0fc5d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/dba294da731146e8b874497334c0fc5d 2024-11-19T21:41:54,693 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/dba294da731146e8b874497334c0fc5d, entries=150, sequenceid=248, filesize=30.4 K 2024-11-19T21:41:54,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c958115d4d0643e1bd1d4af64264f150 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c958115d4d0643e1bd1d4af64264f150 2024-11-19T21:41:54,696 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c958115d4d0643e1bd1d4af64264f150, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T21:41:54,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/abf19c5557ca470e9b7dc2c433230d27 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/abf19c5557ca470e9b7dc2c433230d27 2024-11-19T21:41:54,700 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/abf19c5557ca470e9b7dc2c433230d27, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T21:41:54,701 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a87877009cc80e5fd95e37cd9db304a4 in 1248ms, sequenceid=248, compaction requested=false 2024-11-19T21:41:54,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:54,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:54,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-19T21:41:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-19T21:41:54,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T21:41:54,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-11-19T21:41:54,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4060 sec 2024-11-19T21:41:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:55,180 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:41:55,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:41:55,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111972f8cbfa0e9d4584bb0a79456ad89d02_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052515179/Put/seqid=0 2024-11-19T21:41:55,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742276_1452 (size=14944) 2024-11-19T21:41:55,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052575311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:55,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052575418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T21:41:55,448 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:41:55,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:41:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-19T21:41:55,450 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:41:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T21:41:55,450 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:41:55,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:41:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T21:41:55,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:55,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:55,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:55,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:55,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:55,608 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:55,611 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111972f8cbfa0e9d4584bb0a79456ad89d02_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111972f8cbfa0e9d4584bb0a79456ad89d02_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:55,611 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/61108cebbb51438d978aa938d4b7f8b2, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:55,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/61108cebbb51438d978aa938d4b7f8b2 is 175, key is test_row_0/A:col10/1732052515179/Put/seqid=0 2024-11-19T21:41:55,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742277_1453 (size=39899) 2024-11-19T21:41:55,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052575628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:55,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:55,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T21:41:55,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:55,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:55,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052575938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:56,015 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/61108cebbb51438d978aa938d4b7f8b2 2024-11-19T21:41:56,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c5b840d7574c4e7fbcd0d99f10b1ba9f is 50, key is test_row_0/B:col10/1732052515179/Put/seqid=0 2024-11-19T21:41:56,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742278_1454 (size=12251) 2024-11-19T21:41:56,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:56,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:56,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T21:41:56,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:56,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:56,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c5b840d7574c4e7fbcd0d99f10b1ba9f 2024-11-19T21:41:56,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/cf6b3387c72d497387e4d0ecf3388a14 is 50, key is test_row_0/C:col10/1732052515179/Put/seqid=0 2024-11-19T21:41:56,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742279_1455 (size=12251) 2024-11-19T21:41:56,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:41:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38958 deadline: 1732052576448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 2024-11-19T21:41:56,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:56,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:41:56,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:41:56,507 INFO [Thread-1718 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1718. 2024-11-19T21:41:56,507 DEBUG [Thread-1718 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:56,507 DEBUG [Thread-1718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,507 DEBUG [Thread-1718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,507 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:56,508 INFO [Thread-1720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1720. 2024-11-19T21:41:56,509 DEBUG [Thread-1720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:56,509 DEBUG [Thread-1720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,509 DEBUG [Thread-1720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:56,510 INFO [Thread-1722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1722. 2024-11-19T21:41:56,510 DEBUG [Thread-1722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:56,510 DEBUG [Thread-1722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,510 DEBUG [Thread-1722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:56,517 INFO [Thread-1726 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1726. 2024-11-19T21:41:56,517 DEBUG [Thread-1726 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:56,517 DEBUG [Thread-1726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,517 DEBUG [Thread-1726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:56,518 INFO [Thread-1724 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1724. 2024-11-19T21:41:56,518 DEBUG [Thread-1724 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:56,518 DEBUG [Thread-1724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,518 DEBUG [Thread-1724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:56,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:56,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:56,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:56,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T21:41:56,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:41:56,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/cf6b3387c72d497387e4d0ecf3388a14 2024-11-19T21:41:56,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/61108cebbb51438d978aa938d4b7f8b2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/61108cebbb51438d978aa938d4b7f8b2 2024-11-19T21:41:56,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/61108cebbb51438d978aa938d4b7f8b2, entries=200, sequenceid=262, filesize=39.0 K 2024-11-19T21:41:56,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/c5b840d7574c4e7fbcd0d99f10b1ba9f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c5b840d7574c4e7fbcd0d99f10b1ba9f 2024-11-19T21:41:56,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c5b840d7574c4e7fbcd0d99f10b1ba9f, entries=150, sequenceid=262, filesize=12.0 K 2024-11-19T21:41:56,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/cf6b3387c72d497387e4d0ecf3388a14 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/cf6b3387c72d497387e4d0ecf3388a14 2024-11-19T21:41:56,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/cf6b3387c72d497387e4d0ecf3388a14, entries=150, sequenceid=262, filesize=12.0 K 2024-11-19T21:41:56,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a87877009cc80e5fd95e37cd9db304a4 in 1675ms, sequenceid=262, compaction requested=true 2024-11-19T21:41:56,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:56,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:41:56,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:56,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:41:56,856 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:56,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:56,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a87877009cc80e5fd95e37cd9db304a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:41:56,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:56,856 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:56,857 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:56,857 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:56,857 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/A is initiating minor compaction (all files) 2024-11-19T21:41:56,857 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/B is initiating minor compaction (all files) 2024-11-19T21:41:56,857 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/B in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,857 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/A in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,857 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/75c09db68ab7448bad7f054a98477426, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/dba294da731146e8b874497334c0fc5d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/61108cebbb51438d978aa938d4b7f8b2] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=100.2 K 2024-11-19T21:41:56,857 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/39334c147897408b9298a150150fbb49, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c958115d4d0643e1bd1d4af64264f150, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c5b840d7574c4e7fbcd0d99f10b1ba9f] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=36.2 K 2024-11-19T21:41:56,857 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,857 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/75c09db68ab7448bad7f054a98477426, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/dba294da731146e8b874497334c0fc5d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/61108cebbb51438d978aa938d4b7f8b2] 2024-11-19T21:41:56,858 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 39334c147897408b9298a150150fbb49, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732052511640 2024-11-19T21:41:56,858 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c09db68ab7448bad7f054a98477426, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732052511640 2024-11-19T21:41:56,858 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c958115d4d0643e1bd1d4af64264f150, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732052511800 2024-11-19T21:41:56,858 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting dba294da731146e8b874497334c0fc5d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732052511800 2024-11-19T21:41:56,858 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61108cebbb51438d978aa938d4b7f8b2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732052513995 2024-11-19T21:41:56,858 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c5b840d7574c4e7fbcd0d99f10b1ba9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732052513995 2024-11-19T21:41:56,865 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:56,867 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:56,868 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/a0be34c6ed2b4fb8a18b18a9dfd0b064 is 50, key is test_row_0/B:col10/1732052515179/Put/seqid=0 2024-11-19T21:41:56,868 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119880724228e9a4921afbf6cc0c61e8ec5_a87877009cc80e5fd95e37cd9db304a4 store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:56,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742280_1456 (size=12899) 2024-11-19T21:41:56,872 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119880724228e9a4921afbf6cc0c61e8ec5_a87877009cc80e5fd95e37cd9db304a4, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:56,872 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119880724228e9a4921afbf6cc0c61e8ec5_a87877009cc80e5fd95e37cd9db304a4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:56,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742281_1457 (size=4469) 2024-11-19T21:41:56,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T21:41:56,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:56,981 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:41:56,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f7f34057c16647a9af042e9c3fdf9ad7_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052515274/Put/seqid=0 2024-11-19T21:41:56,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742282_1458 (size=12454) 2024-11-19T21:41:57,196 WARN [regionserver/707262ffa86d:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-19T21:41:57,201 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/6f2e8caaf87941278461d43a4ea6827a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/1f9a17e63eeb40fa827b1916c05f5be2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f7b44d1833c6464682d0bafd2880f12a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/d656ef1c60ca4d6f95311edaa3dc78aa, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/68a18cf481b8408a82efb2a2822c1a80, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/597fea4fe6f746c4b220cb627ec11660, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c1cf2bef643e418a8c5bb42d13741b71, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/020db67e573a41f79a11371e0a45fbae, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/edec103fb2d24437bb52889f0d3aa584, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/4f0e26c7d771470a8569724c79163679, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adac64c05a0141b6bd65e4599e9e1f28, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/bc26f2b202fd4108bdee113dbcefe61e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adf42a9f954f4d24a021886f54e02101, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/87c89f2d9b994bccabf7d346d10b17ed, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/81060eea71cb404f90774774e6656224, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c517e8b2974e4fb8a92805a239e94100] to archive 2024-11-19T21:41:57,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:41:57,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/6f2e8caaf87941278461d43a4ea6827a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/6f2e8caaf87941278461d43a4ea6827a 2024-11-19T21:41:57,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/1f9a17e63eeb40fa827b1916c05f5be2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/1f9a17e63eeb40fa827b1916c05f5be2 2024-11-19T21:41:57,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f7b44d1833c6464682d0bafd2880f12a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f7b44d1833c6464682d0bafd2880f12a 2024-11-19T21:41:57,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/d656ef1c60ca4d6f95311edaa3dc78aa to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/d656ef1c60ca4d6f95311edaa3dc78aa 2024-11-19T21:41:57,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/68a18cf481b8408a82efb2a2822c1a80 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/68a18cf481b8408a82efb2a2822c1a80 2024-11-19T21:41:57,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/597fea4fe6f746c4b220cb627ec11660 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/597fea4fe6f746c4b220cb627ec11660 2024-11-19T21:41:57,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c1cf2bef643e418a8c5bb42d13741b71 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c1cf2bef643e418a8c5bb42d13741b71 2024-11-19T21:41:57,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/020db67e573a41f79a11371e0a45fbae to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/020db67e573a41f79a11371e0a45fbae 2024-11-19T21:41:57,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/edec103fb2d24437bb52889f0d3aa584 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/edec103fb2d24437bb52889f0d3aa584 2024-11-19T21:41:57,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/4f0e26c7d771470a8569724c79163679 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/4f0e26c7d771470a8569724c79163679 2024-11-19T21:41:57,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adac64c05a0141b6bd65e4599e9e1f28 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adac64c05a0141b6bd65e4599e9e1f28 2024-11-19T21:41:57,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/bc26f2b202fd4108bdee113dbcefe61e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/bc26f2b202fd4108bdee113dbcefe61e 2024-11-19T21:41:57,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adf42a9f954f4d24a021886f54e02101 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/adf42a9f954f4d24a021886f54e02101 2024-11-19T21:41:57,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/87c89f2d9b994bccabf7d346d10b17ed to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/87c89f2d9b994bccabf7d346d10b17ed 2024-11-19T21:41:57,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/81060eea71cb404f90774774e6656224 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/81060eea71cb404f90774774e6656224 2024-11-19T21:41:57,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c517e8b2974e4fb8a92805a239e94100 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/c517e8b2974e4fb8a92805a239e94100 2024-11-19T21:41:57,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/6b2cf996aa844042a88dbc7e2c823fcd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a3dda099e96d44fa8a485d8d7c3857d6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/2b27e748675448d19660d4cd2c59a24f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d5a772313d96487783016f2b172c9c66, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/3b35ddd6c46e46c2b3a76f37d7739022, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/f23f367a395b4d93b953936261c7b8d8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/cc7774015e2649649e4a6e7999e51e3c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d020fcfa99ea4ae0a858a7903bfe9414, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/43139c79d1c24fa2a554fac74fc1d952, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/620e299daed5418696ad34f49d265fef, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c4020ec36bfa4321b10beb687fdd2d7e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/bcc70fd412254bdaa48df24f1233b694, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c7e8b11de4c14599a5606e51ae4e1089, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a227b02e19094b539c11ac0a55562ff6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/50846bbc379b4a5f8986ba757a89d2c3, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/dc6dec65d5a94df2a827c473449e3d9f] to archive 2024-11-19T21:41:57,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:41:57,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/6b2cf996aa844042a88dbc7e2c823fcd to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/6b2cf996aa844042a88dbc7e2c823fcd 2024-11-19T21:41:57,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a3dda099e96d44fa8a485d8d7c3857d6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a3dda099e96d44fa8a485d8d7c3857d6 2024-11-19T21:41:57,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/2b27e748675448d19660d4cd2c59a24f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/2b27e748675448d19660d4cd2c59a24f 2024-11-19T21:41:57,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d5a772313d96487783016f2b172c9c66 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d5a772313d96487783016f2b172c9c66 2024-11-19T21:41:57,230 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/3b35ddd6c46e46c2b3a76f37d7739022 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/3b35ddd6c46e46c2b3a76f37d7739022 2024-11-19T21:41:57,231 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/f23f367a395b4d93b953936261c7b8d8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/f23f367a395b4d93b953936261c7b8d8 2024-11-19T21:41:57,233 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/cc7774015e2649649e4a6e7999e51e3c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/cc7774015e2649649e4a6e7999e51e3c 2024-11-19T21:41:57,234 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d020fcfa99ea4ae0a858a7903bfe9414 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/d020fcfa99ea4ae0a858a7903bfe9414 2024-11-19T21:41:57,236 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/43139c79d1c24fa2a554fac74fc1d952 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/43139c79d1c24fa2a554fac74fc1d952 2024-11-19T21:41:57,237 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/620e299daed5418696ad34f49d265fef to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/620e299daed5418696ad34f49d265fef 2024-11-19T21:41:57,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c4020ec36bfa4321b10beb687fdd2d7e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c4020ec36bfa4321b10beb687fdd2d7e 2024-11-19T21:41:57,240 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/bcc70fd412254bdaa48df24f1233b694 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/bcc70fd412254bdaa48df24f1233b694 2024-11-19T21:41:57,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c7e8b11de4c14599a5606e51ae4e1089 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c7e8b11de4c14599a5606e51ae4e1089 2024-11-19T21:41:57,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a227b02e19094b539c11ac0a55562ff6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a227b02e19094b539c11ac0a55562ff6 2024-11-19T21:41:57,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/50846bbc379b4a5f8986ba757a89d2c3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/50846bbc379b4a5f8986ba757a89d2c3 2024-11-19T21:41:57,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/dc6dec65d5a94df2a827c473449e3d9f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/dc6dec65d5a94df2a827c473449e3d9f 2024-11-19T21:41:57,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0268c4cf6abf471bac2dcf4440212478, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/4c782d76b2d141e881945813aeb7728c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/ff80eb089e7f405ab8ab77c9ff522fe5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/51bef94bdc0942498e16caab08ca4c00, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64a8158310244dd9a7513dc83d735404, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/489337d91efb4a7c93e731e3dc5519ac, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/1b79f3106fbc4327a3d4b7409798e6a8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b18e2695595b496c9805a8b89ae7db8c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/a81f8a9b21f24500adbb23eb1099807f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/7228f159eedf4d4390ca079f46edca8a, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0e012f5056b344488e144f75b71e04af, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/9083c3ca2c6240a4952116b3741a9a98, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/e6ebb711b649424798e2f058bb61c998, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/c48dbb36c23f4adb9ca19fad4ff4e80f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/dd0648ef34a445879c4f26a7bad839ea, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/fcb83d33cc0b423bafd096ea82931c52] to archive 2024-11-19T21:41:57,251 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:41:57,252 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0268c4cf6abf471bac2dcf4440212478 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0268c4cf6abf471bac2dcf4440212478 2024-11-19T21:41:57,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/4c782d76b2d141e881945813aeb7728c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/4c782d76b2d141e881945813aeb7728c 2024-11-19T21:41:57,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/ff80eb089e7f405ab8ab77c9ff522fe5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/ff80eb089e7f405ab8ab77c9ff522fe5 2024-11-19T21:41:57,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/51bef94bdc0942498e16caab08ca4c00 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/51bef94bdc0942498e16caab08ca4c00 2024-11-19T21:41:57,255 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64a8158310244dd9a7513dc83d735404 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64a8158310244dd9a7513dc83d735404 2024-11-19T21:41:57,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/489337d91efb4a7c93e731e3dc5519ac to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/489337d91efb4a7c93e731e3dc5519ac 2024-11-19T21:41:57,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/1b79f3106fbc4327a3d4b7409798e6a8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/1b79f3106fbc4327a3d4b7409798e6a8 2024-11-19T21:41:57,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b18e2695595b496c9805a8b89ae7db8c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b18e2695595b496c9805a8b89ae7db8c 2024-11-19T21:41:57,258 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/a81f8a9b21f24500adbb23eb1099807f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/a81f8a9b21f24500adbb23eb1099807f 2024-11-19T21:41:57,259 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/7228f159eedf4d4390ca079f46edca8a to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/7228f159eedf4d4390ca079f46edca8a 2024-11-19T21:41:57,260 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0e012f5056b344488e144f75b71e04af to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/0e012f5056b344488e144f75b71e04af 2024-11-19T21:41:57,260 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/9083c3ca2c6240a4952116b3741a9a98 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/9083c3ca2c6240a4952116b3741a9a98 2024-11-19T21:41:57,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/e6ebb711b649424798e2f058bb61c998 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/e6ebb711b649424798e2f058bb61c998 2024-11-19T21:41:57,262 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/c48dbb36c23f4adb9ca19fad4ff4e80f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/c48dbb36c23f4adb9ca19fad4ff4e80f 2024-11-19T21:41:57,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/dd0648ef34a445879c4f26a7bad839ea to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/dd0648ef34a445879c4f26a7bad839ea 2024-11-19T21:41:57,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/707262ffa86d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/fcb83d33cc0b423bafd096ea82931c52 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/fcb83d33cc0b423bafd096ea82931c52 2024-11-19T21:41:57,274 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/a0be34c6ed2b4fb8a18b18a9dfd0b064 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a0be34c6ed2b4fb8a18b18a9dfd0b064 2024-11-19T21:41:57,276 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#A#compaction#387 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:57,276 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/7f628cfbdc164054a7233f75fb2407e9 is 175, key is test_row_0/A:col10/1732052515179/Put/seqid=0 2024-11-19T21:41:57,279 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/B of a87877009cc80e5fd95e37cd9db304a4 into a0be34c6ed2b4fb8a18b18a9dfd0b064(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:57,279 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:57,279 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/B, priority=13, startTime=1732052516856; duration=0sec 2024-11-19T21:41:57,279 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:41:57,279 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:B 2024-11-19T21:41:57,280 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:41:57,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742283_1459 (size=31308) 2024-11-19T21:41:57,281 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:41:57,281 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): a87877009cc80e5fd95e37cd9db304a4/C is initiating minor compaction (all files) 2024-11-19T21:41:57,281 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a87877009cc80e5fd95e37cd9db304a4/C in TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:57,281 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/6a235b1131294ad687298b79bd29a505, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/abf19c5557ca470e9b7dc2c433230d27, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/cf6b3387c72d497387e4d0ecf3388a14] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp, totalSize=36.2 K 2024-11-19T21:41:57,281 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a235b1131294ad687298b79bd29a505, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732052511640 2024-11-19T21:41:57,282 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting abf19c5557ca470e9b7dc2c433230d27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732052511800 2024-11-19T21:41:57,282 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting cf6b3387c72d497387e4d0ecf3388a14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732052513995 2024-11-19T21:41:57,290 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a87877009cc80e5fd95e37cd9db304a4#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:41:57,291 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/9d7a53b291fb4930a823d102212d4175 is 50, key is test_row_0/C:col10/1732052515179/Put/seqid=0 2024-11-19T21:41:57,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742284_1460 (size=12354) 2024-11-19T21:41:57,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:41:57,403 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f7f34057c16647a9af042e9c3fdf9ad7_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f7f34057c16647a9af042e9c3fdf9ad7_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/f5e6584a75f4444eae3bad31a39411e0, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:41:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/f5e6584a75f4444eae3bad31a39411e0 is 175, key is test_row_0/A:col10/1732052515274/Put/seqid=0 2024-11-19T21:41:57,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742285_1461 (size=31255) 2024-11-19T21:41:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:41:57,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. as already flushing 2024-11-19T21:41:57,472 INFO [Thread-1715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1715. 2024-11-19T21:41:57,473 DEBUG [Thread-1715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:41:57,473 DEBUG [Thread-1715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:57,473 DEBUG [Thread-1715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:41:57,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:41:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T21:41:57,692 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/7f628cfbdc164054a7233f75fb2407e9 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/7f628cfbdc164054a7233f75fb2407e9 2024-11-19T21:41:57,699 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/A of a87877009cc80e5fd95e37cd9db304a4 into 7f628cfbdc164054a7233f75fb2407e9(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:57,699 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:57,699 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/A, priority=13, startTime=1732052516855; duration=0sec 2024-11-19T21:41:57,699 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:57,699 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:A 2024-11-19T21:41:57,700 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/9d7a53b291fb4930a823d102212d4175 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/9d7a53b291fb4930a823d102212d4175 2024-11-19T21:41:57,703 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a87877009cc80e5fd95e37cd9db304a4/C of a87877009cc80e5fd95e37cd9db304a4 into 9d7a53b291fb4930a823d102212d4175(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:41:57,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:57,703 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., storeName=a87877009cc80e5fd95e37cd9db304a4/C, priority=13, startTime=1732052516856; duration=0sec 2024-11-19T21:41:57,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:41:57,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a87877009cc80e5fd95e37cd9db304a4:C 2024-11-19T21:41:57,810 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/f5e6584a75f4444eae3bad31a39411e0 2024-11-19T21:41:57,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/232fe555f9ce48c5a2023943d9c1ca94 is 50, key is test_row_0/B:col10/1732052515274/Put/seqid=0 2024-11-19T21:41:57,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742286_1462 (size=12301) 2024-11-19T21:41:58,229 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/232fe555f9ce48c5a2023943d9c1ca94 2024-11-19T21:41:58,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/64281a80b56e47a1998c798338177a96 is 50, key is test_row_0/C:col10/1732052515274/Put/seqid=0 2024-11-19T21:41:58,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742287_1463 (size=12301) 2024-11-19T21:41:58,651 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/64281a80b56e47a1998c798338177a96 2024-11-19T21:41:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/f5e6584a75f4444eae3bad31a39411e0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f5e6584a75f4444eae3bad31a39411e0 2024-11-19T21:41:58,667 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f5e6584a75f4444eae3bad31a39411e0, entries=150, sequenceid=286, filesize=30.5 K 2024-11-19T21:41:58,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/232fe555f9ce48c5a2023943d9c1ca94 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/232fe555f9ce48c5a2023943d9c1ca94 2024-11-19T21:41:58,672 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/232fe555f9ce48c5a2023943d9c1ca94, entries=150, sequenceid=286, filesize=12.0 K 2024-11-19T21:41:58,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/64281a80b56e47a1998c798338177a96 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64281a80b56e47a1998c798338177a96 2024-11-19T21:41:58,677 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64281a80b56e47a1998c798338177a96, entries=150, sequenceid=286, filesize=12.0 K 2024-11-19T21:41:58,678 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for a87877009cc80e5fd95e37cd9db304a4 in 1698ms, sequenceid=286, compaction requested=false 2024-11-19T21:41:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for a87877009cc80e5fd95e37cd9db304a4: 2024-11-19T21:41:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:41:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-19T21:41:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-19T21:41:58,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T21:41:58,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2300 sec 2024-11-19T21:41:58,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.2350 sec 2024-11-19T21:41:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T21:41:59,629 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:00,213 INFO [Thread-1707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1707. 2024-11-19T21:42:00,213 DEBUG [Thread-1707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:00,213 DEBUG [Thread-1707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,213 INFO [Thread-1713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1713. 2024-11-19T21:42:00,213 DEBUG [Thread-1707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,213 DEBUG [Thread-1713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:00,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:00,213 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,214 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:00,260 INFO [Thread-1709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1709. 2024-11-19T21:42:00,260 DEBUG [Thread-1709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:00,260 DEBUG [Thread-1709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,260 DEBUG [Thread-1709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:00,311 INFO [Thread-1711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1711. 2024-11-19T21:42:00,311 DEBUG [Thread-1711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:00,311 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,311 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 171 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1479 2024-11-19T21:42:00,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4437 rows 2024-11-19T21:42:00,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1487 2024-11-19T21:42:00,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4460 rows 2024-11-19T21:42:00,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1488 2024-11-19T21:42:00,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4464 rows 2024-11-19T21:42:00,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1493 2024-11-19T21:42:00,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4469 rows 2024-11-19T21:42:00,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1492 2024-11-19T21:42:00,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4466 rows 2024-11-19T21:42:00,313 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:42:00,313 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:42:00,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:00,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:00,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T21:42:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T21:42:00,319 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052520319"}]},"ts":"1732052520319"} 2024-11-19T21:42:00,320 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T21:42:00,320 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T21:42:00,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T21:42:00,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, UNASSIGN}] 2024-11-19T21:42:00,323 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, UNASSIGN 2024-11-19T21:42:00,324 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=a87877009cc80e5fd95e37cd9db304a4, regionState=CLOSING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:42:00,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, UNASSIGN because future has completed 2024-11-19T21:42:00,325 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T21:42:00,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:42:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T21:42:00,477 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:00,478 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T21:42:00,478 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing a87877009cc80e5fd95e37cd9db304a4, disabling compactions & flushes 2024-11-19T21:42:00,478 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:42:00,478 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:42:00,478 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. after waiting 0 ms 2024-11-19T21:42:00,478 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:42:00,478 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing a87877009cc80e5fd95e37cd9db304a4 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T21:42:00,490 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ff768e1aeb9a4925896b2d50bca215eb_a87877009cc80e5fd95e37cd9db304a4 is 50, key is test_row_0/A:col10/1732052520258/Put/seqid=0 2024-11-19T21:42:00,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742288_1464 (size=12454) 2024-11-19T21:42:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T21:42:00,896 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:00,906 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ff768e1aeb9a4925896b2d50bca215eb_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ff768e1aeb9a4925896b2d50bca215eb_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:00,908 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/8c1b1f27cd544040bdc4540383eb1f02, store: [table=TestAcidGuarantees family=A region=a87877009cc80e5fd95e37cd9db304a4] 2024-11-19T21:42:00,909 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/8c1b1f27cd544040bdc4540383eb1f02 is 175, key is test_row_0/A:col10/1732052520258/Put/seqid=0 2024-11-19T21:42:00,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742289_1465 (size=31255) 2024-11-19T21:42:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T21:42:01,314 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/8c1b1f27cd544040bdc4540383eb1f02 2024-11-19T21:42:01,322 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/f191e92f88f4457981e02852f5e1488b is 50, key is test_row_0/B:col10/1732052520258/Put/seqid=0 2024-11-19T21:42:01,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742290_1466 (size=12301) 2024-11-19T21:42:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T21:42:01,727 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/f191e92f88f4457981e02852f5e1488b 2024-11-19T21:42:01,778 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/b7edf53b7b2941d0b296f815182c6771 is 50, key is test_row_0/C:col10/1732052520258/Put/seqid=0 2024-11-19T21:42:01,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742291_1467 (size=12301) 2024-11-19T21:42:02,184 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/b7edf53b7b2941d0b296f815182c6771 2024-11-19T21:42:02,193 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/A/8c1b1f27cd544040bdc4540383eb1f02 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/8c1b1f27cd544040bdc4540383eb1f02 2024-11-19T21:42:02,199 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/8c1b1f27cd544040bdc4540383eb1f02, entries=150, sequenceid=297, filesize=30.5 K 2024-11-19T21:42:02,200 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/B/f191e92f88f4457981e02852f5e1488b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/f191e92f88f4457981e02852f5e1488b 2024-11-19T21:42:02,202 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/f191e92f88f4457981e02852f5e1488b, entries=150, sequenceid=297, filesize=12.0 K 2024-11-19T21:42:02,203 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/.tmp/C/b7edf53b7b2941d0b296f815182c6771 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b7edf53b7b2941d0b296f815182c6771 2024-11-19T21:42:02,205 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b7edf53b7b2941d0b296f815182c6771, entries=150, sequenceid=297, filesize=12.0 K 2024-11-19T21:42:02,206 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a87877009cc80e5fd95e37cd9db304a4 in 1728ms, sequenceid=297, compaction requested=true 2024-11-19T21:42:02,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/75c09db68ab7448bad7f054a98477426, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/dba294da731146e8b874497334c0fc5d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/61108cebbb51438d978aa938d4b7f8b2] to archive 2024-11-19T21:42:02,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:42:02,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/75c09db68ab7448bad7f054a98477426 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/75c09db68ab7448bad7f054a98477426 2024-11-19T21:42:02,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/dba294da731146e8b874497334c0fc5d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/dba294da731146e8b874497334c0fc5d 2024-11-19T21:42:02,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/61108cebbb51438d978aa938d4b7f8b2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/61108cebbb51438d978aa938d4b7f8b2 2024-11-19T21:42:02,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/39334c147897408b9298a150150fbb49, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c958115d4d0643e1bd1d4af64264f150, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c5b840d7574c4e7fbcd0d99f10b1ba9f] to archive 2024-11-19T21:42:02,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:42:02,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/39334c147897408b9298a150150fbb49 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/39334c147897408b9298a150150fbb49 2024-11-19T21:42:02,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c958115d4d0643e1bd1d4af64264f150 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c958115d4d0643e1bd1d4af64264f150 2024-11-19T21:42:02,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c5b840d7574c4e7fbcd0d99f10b1ba9f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/c5b840d7574c4e7fbcd0d99f10b1ba9f 2024-11-19T21:42:02,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/6a235b1131294ad687298b79bd29a505, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/abf19c5557ca470e9b7dc2c433230d27, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/cf6b3387c72d497387e4d0ecf3388a14] to archive 2024-11-19T21:42:02,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:42:02,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/6a235b1131294ad687298b79bd29a505 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/6a235b1131294ad687298b79bd29a505 2024-11-19T21:42:02,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/abf19c5557ca470e9b7dc2c433230d27 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/abf19c5557ca470e9b7dc2c433230d27 2024-11-19T21:42:02,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/cf6b3387c72d497387e4d0ecf3388a14 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/cf6b3387c72d497387e4d0ecf3388a14 2024-11-19T21:42:02,223 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/recovered.edits/300.seqid, newMaxSeqId=300, maxSeqId=4 2024-11-19T21:42:02,223 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. 2024-11-19T21:42:02,223 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for a87877009cc80e5fd95e37cd9db304a4: Waiting for close lock at 1732052520478Running coprocessor pre-close hooks at 1732052520478Disabling compacts and flushes for region at 1732052520478Disabling writes for close at 1732052520478Obtaining lock to block concurrent updates at 1732052520478Preparing flush snapshotting stores in a87877009cc80e5fd95e37cd9db304a4 at 1732052520478Finished memstore snapshotting TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732052520479 (+1 ms)Flushing stores of TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4. at 1732052520480 (+1 ms)Flushing a87877009cc80e5fd95e37cd9db304a4/A: creating writer at 1732052520480Flushing mob file a87877009cc80e5fd95e37cd9db304a4/A: appending metadata at 1732052520490 (+10 ms)Flushing mob file a87877009cc80e5fd95e37cd9db304a4/A: closing flushed file at 1732052520490Flushing a87877009cc80e5fd95e37cd9db304a4/A: appending metadata at 1732052520908 (+418 ms)Flushing a87877009cc80e5fd95e37cd9db304a4/A: closing flushed file at 1732052520908Flushing a87877009cc80e5fd95e37cd9db304a4/B: creating writer at 1732052521319 (+411 ms)Flushing a87877009cc80e5fd95e37cd9db304a4/B: appending metadata at 1732052521322 (+3 ms)Flushing a87877009cc80e5fd95e37cd9db304a4/B: closing flushed file at 1732052521322Flushing a87877009cc80e5fd95e37cd9db304a4/C: creating writer at 1732052521773 (+451 ms)Flushing a87877009cc80e5fd95e37cd9db304a4/C: appending metadata at 1732052521778 (+5 ms)Flushing a87877009cc80e5fd95e37cd9db304a4/C: closing flushed file at 1732052521778Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4af2968d: reopening flushed file at 1732052522192 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13b5cbe0: reopening flushed file at 1732052522199 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17632bac: reopening flushed file at 1732052522203 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a87877009cc80e5fd95e37cd9db304a4 in 1728ms, sequenceid=297, compaction requested=true at 1732052522206 (+3 ms)Writing region close event to WAL at 1732052522220 (+14 ms)Running coprocessor post-close hooks at 1732052522223 (+3 ms)Closed at 1732052522223 2024-11-19T21:42:02,226 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,226 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=a87877009cc80e5fd95e37cd9db304a4, regionState=CLOSED 2024-11-19T21:42:02,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:42:02,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T21:42:02,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure a87877009cc80e5fd95e37cd9db304a4, server=707262ffa86d,42821,1732052394005 in 1.9040 sec 2024-11-19T21:42:02,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-19T21:42:02,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a87877009cc80e5fd95e37cd9db304a4, UNASSIGN in 1.9080 sec 2024-11-19T21:42:02,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T21:42:02,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9120 sec 2024-11-19T21:42:02,236 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052522235"}]},"ts":"1732052522235"} 2024-11-19T21:42:02,240 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T21:42:02,240 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T21:42:02,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9260 sec 2024-11-19T21:42:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T21:42:02,469 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:02,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T21:42:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:02,474 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T21:42:02,475 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:02,478 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,482 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/recovered.edits] 2024-11-19T21:42:02,487 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/7f628cfbdc164054a7233f75fb2407e9 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/7f628cfbdc164054a7233f75fb2407e9 2024-11-19T21:42:02,490 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/8c1b1f27cd544040bdc4540383eb1f02 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/8c1b1f27cd544040bdc4540383eb1f02 2024-11-19T21:42:02,491 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f5e6584a75f4444eae3bad31a39411e0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/A/f5e6584a75f4444eae3bad31a39411e0 2024-11-19T21:42:02,495 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/232fe555f9ce48c5a2023943d9c1ca94 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/232fe555f9ce48c5a2023943d9c1ca94 2024-11-19T21:42:02,496 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a0be34c6ed2b4fb8a18b18a9dfd0b064 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/a0be34c6ed2b4fb8a18b18a9dfd0b064 2024-11-19T21:42:02,498 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/f191e92f88f4457981e02852f5e1488b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/B/f191e92f88f4457981e02852f5e1488b 2024-11-19T21:42:02,501 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64281a80b56e47a1998c798338177a96 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/64281a80b56e47a1998c798338177a96 2024-11-19T21:42:02,502 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/9d7a53b291fb4930a823d102212d4175 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/9d7a53b291fb4930a823d102212d4175 2024-11-19T21:42:02,504 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b7edf53b7b2941d0b296f815182c6771 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/C/b7edf53b7b2941d0b296f815182c6771 2024-11-19T21:42:02,508 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/recovered.edits/300.seqid to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4/recovered.edits/300.seqid 2024-11-19T21:42:02,509 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,509 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T21:42:02,509 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T21:42:02,510 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T21:42:02,514 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119045b3fd3629e4a32a5c1ad76d9d0c345_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119045b3fd3629e4a32a5c1ad76d9d0c345_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,516 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119108bacf3519241d78fca794256cbd4bb_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119108bacf3519241d78fca794256cbd4bb_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,518 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111918e6027cdacc46b9adeaff088cf64000_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111918e6027cdacc46b9adeaff088cf64000_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,519 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111942ac82b4efd44dc8b44b9ff0e8349024_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111942ac82b4efd44dc8b44b9ff0e8349024_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,520 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194cb58af773c4492b8147037e193d74c5_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194cb58af773c4492b8147037e193d74c5_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,520 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196be03743656b4d3ebe352590acddd00e_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196be03743656b4d3ebe352590acddd00e_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,521 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111972f8cbfa0e9d4584bb0a79456ad89d02_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111972f8cbfa0e9d4584bb0a79456ad89d02_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,522 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199f8d5afb96d644a5a51c138465326419_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199f8d5afb96d644a5a51c138465326419_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,522 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a253e3749b8e485aa5bf85352b014218_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a253e3749b8e485aa5bf85352b014218_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,523 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a2a50ffb6d0543cca4c1a2e5307b2c7b_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a2a50ffb6d0543cca4c1a2e5307b2c7b_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,524 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119acfa745e4f534e438f47e04abd9a3872_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119acfa745e4f534e438f47e04abd9a3872_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,524 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b24fe985e89647e387c628e4b180a5da_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b24fe985e89647e387c628e4b180a5da_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,525 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d892a00c289845858ddfd581ac83f667_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d892a00c289845858ddfd581ac83f667_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,525 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ec0f2456132f4b9c86f678ddbf8d9659_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ec0f2456132f4b9c86f678ddbf8d9659_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,526 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f7f34057c16647a9af042e9c3fdf9ad7_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f7f34057c16647a9af042e9c3fdf9ad7_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,527 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ff768e1aeb9a4925896b2d50bca215eb_a87877009cc80e5fd95e37cd9db304a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ff768e1aeb9a4925896b2d50bca215eb_a87877009cc80e5fd95e37cd9db304a4 2024-11-19T21:42:02,527 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T21:42:02,529 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:02,530 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T21:42:02,532 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T21:42:02,533 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:02,533 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T21:42:02,533 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732052522533"}]},"ts":"9223372036854775807"} 2024-11-19T21:42:02,535 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T21:42:02,535 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a87877009cc80e5fd95e37cd9db304a4, NAME => 'TestAcidGuarantees,,1732052493349.a87877009cc80e5fd95e37cd9db304a4.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T21:42:02,535 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T21:42:02,535 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732052522535"}]},"ts":"9223372036854775807"} 2024-11-19T21:42:02,537 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T21:42:02,537 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:02,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-19T21:42:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T21:42:02,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T21:42:02,579 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:02,592 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 237) - Thread LEAK? -, OpenFileDescriptor=455 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 422), ProcessCount=11 (was 11), AvailableMemoryMB=5032 (was 5071) 2024-11-19T21:42:02,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=5031 2024-11-19T21:42:02,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T21:42:02,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:42:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:02,602 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T21:42:02,602 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:02,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-19T21:42:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-19T21:42:02,603 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T21:42:02,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742292_1468 (size=959) 2024-11-19T21:42:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-19T21:42:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-19T21:42:03,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b 2024-11-19T21:42:03,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742293_1469 (size=53) 2024-11-19T21:42:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-19T21:42:03,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:42:03,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing abb063485168bef9628960f3efec5fb9, disabling compactions & flushes 2024-11-19T21:42:03,425 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:03,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:03,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. after waiting 0 ms 2024-11-19T21:42:03,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:03,425 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:03,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for abb063485168bef9628960f3efec5fb9: Waiting for close lock at 1732052523425Disabling compacts and flushes for region at 1732052523425Disabling writes for close at 1732052523425Writing region close event to WAL at 1732052523425Closed at 1732052523425 2024-11-19T21:42:03,428 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T21:42:03,428 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732052523428"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732052523428"}]},"ts":"1732052523428"} 2024-11-19T21:42:03,432 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T21:42:03,434 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T21:42:03,435 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052523434"}]},"ts":"1732052523434"} 2024-11-19T21:42:03,437 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T21:42:03,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abb063485168bef9628960f3efec5fb9, ASSIGN}] 2024-11-19T21:42:03,438 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abb063485168bef9628960f3efec5fb9, ASSIGN 2024-11-19T21:42:03,439 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=abb063485168bef9628960f3efec5fb9, ASSIGN; state=OFFLINE, location=707262ffa86d,42821,1732052394005; forceNewPlan=false, retain=false 2024-11-19T21:42:03,590 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=abb063485168bef9628960f3efec5fb9, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:42:03,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T21:42:03,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=abb063485168bef9628960f3efec5fb9, ASSIGN because future has completed 2024-11-19T21:42:03,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:42:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-19T21:42:03,754 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:03,755 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:42:03,756 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,756 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:42:03,756 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,756 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,758 INFO [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,759 INFO [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abb063485168bef9628960f3efec5fb9 columnFamilyName A 2024-11-19T21:42:03,760 DEBUG [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:03,760 INFO [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] regionserver.HStore(327): Store=abb063485168bef9628960f3efec5fb9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:42:03,760 INFO [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,761 INFO [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abb063485168bef9628960f3efec5fb9 columnFamilyName B 2024-11-19T21:42:03,761 DEBUG [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:03,762 INFO [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] regionserver.HStore(327): Store=abb063485168bef9628960f3efec5fb9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:42:03,762 INFO [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,763 INFO [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abb063485168bef9628960f3efec5fb9 columnFamilyName C 2024-11-19T21:42:03,763 DEBUG [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:03,764 INFO [StoreOpener-abb063485168bef9628960f3efec5fb9-1 {}] regionserver.HStore(327): Store=abb063485168bef9628960f3efec5fb9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:42:03,764 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,765 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,765 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,766 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,766 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,767 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:42:03,768 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,770 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:42:03,771 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened abb063485168bef9628960f3efec5fb9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62068848, jitterRate=-0.07510209083557129}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:42:03,771 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:03,772 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for abb063485168bef9628960f3efec5fb9: Running coprocessor pre-open hook at 1732052523757Writing region info on filesystem at 1732052523757Initializing all the Stores at 1732052523757Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052523758 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052523758Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052523758Cleaning up temporary data from old regions at 1732052523766 (+8 ms)Running coprocessor post-open hooks at 1732052523771 (+5 ms)Region opened successfully at 1732052523772 (+1 ms) 2024-11-19T21:42:03,772 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., pid=118, masterSystemTime=1732052523747 2024-11-19T21:42:03,774 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:03,774 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:03,775 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=abb063485168bef9628960f3efec5fb9, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:42:03,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:42:03,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T21:42:03,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 in 184 msec 2024-11-19T21:42:03,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-19T21:42:03,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abb063485168bef9628960f3efec5fb9, ASSIGN in 342 msec 2024-11-19T21:42:03,782 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T21:42:03,783 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052523782"}]},"ts":"1732052523782"} 2024-11-19T21:42:03,784 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T21:42:03,785 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T21:42:03,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-19T21:42:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-19T21:42:04,749 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:04,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558db58b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,751 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@766c4489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,752 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b28188b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,755 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T21:42:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 707262ffa86d,37541,1732052393169 2024-11-19T21:42:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40367ce6 2024-11-19T21:42:04,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T21:42:04,756 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T21:42:04,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7631696d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,758 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a21690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,759 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b0feef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126b8236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,761 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d8ad9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,762 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710b24fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea00808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,764 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41994f10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,765 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c165121, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1589e05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,766 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b15d26a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d47ba30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e46d68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,769 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c65935c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,771 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35caaff3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b23f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,772 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490ad364, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,774 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a92ec91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7d791a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,775 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bebf276, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,776 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4dcd40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b3a94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,778 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233f05da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,779 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@546db457, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c68a1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,780 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,781 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c023c6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,782 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99c0966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5c6054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:04,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:04,784 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4510c2f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:04,785 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f708401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:04,786 DEBUG [Thread-2098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,786 DEBUG [Thread-2100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,786 DEBUG [Thread-2107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,786 DEBUG [Thread-2109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,787 DEBUG [Thread-2102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,788 DEBUG [Thread-2113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,789 DEBUG [Thread-2104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,789 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:04,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,790 DEBUG [Thread-2111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-19T21:42:04,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,792 DEBUG [Thread-2115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T21:42:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,794 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:04,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,795 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,797 DEBUG [Thread-2096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,799 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:04,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2] 2024-11-19T21:42:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:04,803 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:04,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:42:04,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/b8aef36950ce44c8b069d8ea8d5be576 is 50, key is test_row_0/A:col10/1732052524800/Put/seqid=0 2024-11-19T21:42:04,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742294_1470 (size=16681) 2024-11-19T21:42:04,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/b8aef36950ce44c8b069d8ea8d5be576 2024-11-19T21:42:04,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052584843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052584844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052584845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/583590b783cb47a0928d78f34a57e29d is 50, key is test_row_0/B:col10/1732052524800/Put/seqid=0 2024-11-19T21:42:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:04,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052584846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:04,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052584847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:04,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742295_1471 (size=12001) 2024-11-19T21:42:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T21:42:04,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T21:42:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:04,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:04,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:04,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:04,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:04,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052584958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052584958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:04,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052584958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052584958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052584958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T21:42:05,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T21:42:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:05,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052585168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052585168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052585168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052585168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052585178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T21:42:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:05,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/583590b783cb47a0928d78f34a57e29d 2024-11-19T21:42:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/b21016646e3043dd960d469fb8d80414 is 50, key is test_row_0/C:col10/1732052524800/Put/seqid=0 2024-11-19T21:42:05,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742296_1472 (size=12001) 2024-11-19T21:42:05,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T21:42:05,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:05,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:05,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:05,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T21:42:05,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052585477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052585477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052585478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052585478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052585488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:05,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T21:42:05,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:05,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:05,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:05,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:05,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/b21016646e3043dd960d469fb8d80414 2024-11-19T21:42:05,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/b8aef36950ce44c8b069d8ea8d5be576 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b8aef36950ce44c8b069d8ea8d5be576 2024-11-19T21:42:05,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b8aef36950ce44c8b069d8ea8d5be576, entries=250, sequenceid=12, filesize=16.3 K 2024-11-19T21:42:05,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/583590b783cb47a0928d78f34a57e29d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/583590b783cb47a0928d78f34a57e29d 2024-11-19T21:42:05,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/583590b783cb47a0928d78f34a57e29d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T21:42:05,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/b21016646e3043dd960d469fb8d80414 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b21016646e3043dd960d469fb8d80414 2024-11-19T21:42:05,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b21016646e3043dd960d469fb8d80414, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T21:42:05,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for abb063485168bef9628960f3efec5fb9 in 900ms, sequenceid=12, compaction requested=false 2024-11-19T21:42:05,703 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T21:42:05,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:05,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T21:42:05,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:05,719 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:42:05,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/4b255e7814624b51a5e1addf7230e5fa is 50, key is test_row_0/A:col10/1732052524846/Put/seqid=0 2024-11-19T21:42:05,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742297_1473 (size=12001) 2024-11-19T21:42:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T21:42:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:05,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:06,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052585999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052586000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052586004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052586004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052586005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052586108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052586108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052586118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052586118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052586118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,133 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/4b255e7814624b51a5e1addf7230e5fa 2024-11-19T21:42:06,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/e585a4ef35ed4352b926c9dcc2dde392 is 50, key is test_row_0/B:col10/1732052524846/Put/seqid=0 2024-11-19T21:42:06,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742298_1474 (size=12001) 2024-11-19T21:42:06,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052586317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052586318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052586328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052586328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052586328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,543 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/e585a4ef35ed4352b926c9dcc2dde392 2024-11-19T21:42:06,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/14d7a9519e9d464f91f57a81ea80cce0 is 50, key is test_row_0/C:col10/1732052524846/Put/seqid=0 2024-11-19T21:42:06,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742299_1475 (size=12001) 2024-11-19T21:42:06,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052586628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052586628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052586638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052586638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052586638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T21:42:06,953 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/14d7a9519e9d464f91f57a81ea80cce0 2024-11-19T21:42:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/4b255e7814624b51a5e1addf7230e5fa as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4b255e7814624b51a5e1addf7230e5fa 2024-11-19T21:42:06,959 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4b255e7814624b51a5e1addf7230e5fa, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T21:42:06,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/e585a4ef35ed4352b926c9dcc2dde392 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e585a4ef35ed4352b926c9dcc2dde392 2024-11-19T21:42:06,962 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e585a4ef35ed4352b926c9dcc2dde392, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T21:42:06,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/14d7a9519e9d464f91f57a81ea80cce0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/14d7a9519e9d464f91f57a81ea80cce0 2024-11-19T21:42:06,965 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/14d7a9519e9d464f91f57a81ea80cce0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T21:42:06,966 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for abb063485168bef9628960f3efec5fb9 in 1247ms, sequenceid=37, compaction requested=false 2024-11-19T21:42:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-19T21:42:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-19T21:42:06,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T21:42:06,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-11-19T21:42:06,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.1790 sec 2024-11-19T21:42:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:07,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:42:07,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/7e51ab78cd0e43498216e300940b8c4c is 50, key is test_row_0/A:col10/1732052526004/Put/seqid=0 2024-11-19T21:42:07,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742300_1476 (size=14341) 2024-11-19T21:42:07,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052587182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052587182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052587183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052587184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052587190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052587298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052587298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052587299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052587299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052587299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052587517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052587518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052587518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052587518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052587518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/7e51ab78cd0e43498216e300940b8c4c 2024-11-19T21:42:07,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/c18e117beaba460db9080d1d42abcb0c is 50, key is test_row_0/B:col10/1732052526004/Put/seqid=0 2024-11-19T21:42:07,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742301_1477 (size=12001) 2024-11-19T21:42:07,741 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T21:42:07,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052587828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052587828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052587828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052587838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052587838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:07,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/c18e117beaba460db9080d1d42abcb0c 2024-11-19T21:42:07,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/a4621f1dd323442780e0ea96718c3fab is 50, key is test_row_0/C:col10/1732052526004/Put/seqid=0 2024-11-19T21:42:07,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742302_1478 (size=12001) 2024-11-19T21:42:08,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052588338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052588348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:08,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052588348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:08,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052588348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:08,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052588358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:08,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/a4621f1dd323442780e0ea96718c3fab 2024-11-19T21:42:08,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/7e51ab78cd0e43498216e300940b8c4c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/7e51ab78cd0e43498216e300940b8c4c 2024-11-19T21:42:08,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/7e51ab78cd0e43498216e300940b8c4c, entries=200, sequenceid=49, filesize=14.0 K 2024-11-19T21:42:08,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/c18e117beaba460db9080d1d42abcb0c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/c18e117beaba460db9080d1d42abcb0c 2024-11-19T21:42:08,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/c18e117beaba460db9080d1d42abcb0c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T21:42:08,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/a4621f1dd323442780e0ea96718c3fab as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/a4621f1dd323442780e0ea96718c3fab 2024-11-19T21:42:08,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/a4621f1dd323442780e0ea96718c3fab, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T21:42:08,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for abb063485168bef9628960f3efec5fb9 in 1242ms, sequenceid=49, compaction requested=true 2024-11-19T21:42:08,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:08,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:08,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:08,382 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:08,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:08,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:08,382 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:08,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:08,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:08,382 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:08,382 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:08,382 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/B is initiating minor compaction (all files) 2024-11-19T21:42:08,382 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/A is initiating minor compaction (all files) 2024-11-19T21:42:08,382 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/B in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:08,382 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/A in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:08,383 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b8aef36950ce44c8b069d8ea8d5be576, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4b255e7814624b51a5e1addf7230e5fa, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/7e51ab78cd0e43498216e300940b8c4c] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=42.0 K 2024-11-19T21:42:08,383 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/583590b783cb47a0928d78f34a57e29d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e585a4ef35ed4352b926c9dcc2dde392, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/c18e117beaba460db9080d1d42abcb0c] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=35.2 K 2024-11-19T21:42:08,383 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 583590b783cb47a0928d78f34a57e29d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732052524795 2024-11-19T21:42:08,383 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8aef36950ce44c8b069d8ea8d5be576, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732052524795 2024-11-19T21:42:08,383 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b255e7814624b51a5e1addf7230e5fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732052524839 2024-11-19T21:42:08,383 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e585a4ef35ed4352b926c9dcc2dde392, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732052524839 2024-11-19T21:42:08,383 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e51ab78cd0e43498216e300940b8c4c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732052526002 2024-11-19T21:42:08,383 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c18e117beaba460db9080d1d42abcb0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732052526003 2024-11-19T21:42:08,388 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:08,388 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/4dbfca20c1b041218b92f449ee5a9010 is 50, key is test_row_0/B:col10/1732052526004/Put/seqid=0 2024-11-19T21:42:08,391 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#A#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:08,391 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/3a15f86c449947108422ebcdcd700c62 is 50, key is test_row_0/A:col10/1732052526004/Put/seqid=0 2024-11-19T21:42:08,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742303_1479 (size=12104) 2024-11-19T21:42:08,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742304_1480 (size=12104) 2024-11-19T21:42:08,796 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/4dbfca20c1b041218b92f449ee5a9010 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/4dbfca20c1b041218b92f449ee5a9010 2024-11-19T21:42:08,799 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/B of abb063485168bef9628960f3efec5fb9 into 4dbfca20c1b041218b92f449ee5a9010(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:08,799 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:08,799 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/B, priority=13, startTime=1732052528382; duration=0sec 2024-11-19T21:42:08,799 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:08,799 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:B 2024-11-19T21:42:08,799 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:08,799 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/3a15f86c449947108422ebcdcd700c62 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/3a15f86c449947108422ebcdcd700c62 2024-11-19T21:42:08,800 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:08,800 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/C is initiating minor compaction (all files) 2024-11-19T21:42:08,800 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/C in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:08,800 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b21016646e3043dd960d469fb8d80414, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/14d7a9519e9d464f91f57a81ea80cce0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/a4621f1dd323442780e0ea96718c3fab] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=35.2 K 2024-11-19T21:42:08,800 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b21016646e3043dd960d469fb8d80414, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732052524795 2024-11-19T21:42:08,800 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d7a9519e9d464f91f57a81ea80cce0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732052524839 2024-11-19T21:42:08,801 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a4621f1dd323442780e0ea96718c3fab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732052526003 2024-11-19T21:42:08,803 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/A of abb063485168bef9628960f3efec5fb9 into 3a15f86c449947108422ebcdcd700c62(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:08,803 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:08,803 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/A, priority=13, startTime=1732052528382; duration=0sec 2024-11-19T21:42:08,803 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:08,803 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:A 2024-11-19T21:42:08,807 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:08,807 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/c96dbeb9fe29474aa7651dda7e3b1bd6 is 50, key is test_row_0/C:col10/1732052526004/Put/seqid=0 2024-11-19T21:42:08,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742305_1481 (size=12104) 2024-11-19T21:42:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T21:42:08,969 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:08,969 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-19T21:42:08,971 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T21:42:08,971 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:08,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T21:42:09,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T21:42:09,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:09,123 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:42:09,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/23d4bb84103a4f5fbfccb76b8c927b42 is 50, key is test_row_0/A:col10/1732052527182/Put/seqid=0 2024-11-19T21:42:09,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742306_1482 (size=12001) 2024-11-19T21:42:09,214 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/c96dbeb9fe29474aa7651dda7e3b1bd6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/c96dbeb9fe29474aa7651dda7e3b1bd6 2024-11-19T21:42:09,218 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/C of abb063485168bef9628960f3efec5fb9 into c96dbeb9fe29474aa7651dda7e3b1bd6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:09,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:09,218 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/C, priority=13, startTime=1732052528382; duration=0sec 2024-11-19T21:42:09,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:09,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:C 2024-11-19T21:42:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T21:42:09,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:09,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052589371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052589372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052589376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052589377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052589378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,385 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in abb063485168bef9628960f3efec5fb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052589488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052589488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052589488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052589488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,532 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/23d4bb84103a4f5fbfccb76b8c927b42 2024-11-19T21:42:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/e064f15e7af54ed7bfb2cbae2875389e is 50, key is test_row_0/B:col10/1732052527182/Put/seqid=0 2024-11-19T21:42:09,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742307_1483 (size=12001) 2024-11-19T21:42:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T21:42:09,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052589698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052589698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052589698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052589698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:09,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:09,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:09,944 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/e064f15e7af54ed7bfb2cbae2875389e 2024-11-19T21:42:09,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/df3da3c56f68476d8349f993cf104679 is 50, key is test_row_0/C:col10/1732052527182/Put/seqid=0 2024-11-19T21:42:09,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742308_1484 (size=12001) 2024-11-19T21:42:10,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052590008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052590008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052590008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052590008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T21:42:10,367 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/df3da3c56f68476d8349f993cf104679 2024-11-19T21:42:10,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/23d4bb84103a4f5fbfccb76b8c927b42 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/23d4bb84103a4f5fbfccb76b8c927b42 2024-11-19T21:42:10,377 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/23d4bb84103a4f5fbfccb76b8c927b42, entries=150, sequenceid=75, filesize=11.7 K 2024-11-19T21:42:10,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/e064f15e7af54ed7bfb2cbae2875389e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e064f15e7af54ed7bfb2cbae2875389e 2024-11-19T21:42:10,383 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e064f15e7af54ed7bfb2cbae2875389e, entries=150, sequenceid=75, filesize=11.7 K 2024-11-19T21:42:10,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/df3da3c56f68476d8349f993cf104679 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/df3da3c56f68476d8349f993cf104679 2024-11-19T21:42:10,392 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/df3da3c56f68476d8349f993cf104679, entries=150, sequenceid=75, filesize=11.7 K 2024-11-19T21:42:10,393 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for abb063485168bef9628960f3efec5fb9 in 1270ms, sequenceid=75, compaction requested=false 2024-11-19T21:42:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-19T21:42:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-19T21:42:10,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T21:42:10,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4230 sec 2024-11-19T21:42:10,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4280 sec 2024-11-19T21:42:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:10,524 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:10,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:42:10,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/6e2ca4ab25e6469194a28dc426cf0945 is 50, key is test_row_0/A:col10/1732052529374/Put/seqid=0 2024-11-19T21:42:10,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742309_1485 (size=12001) 2024-11-19T21:42:10,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052590563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052590568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052590584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052590584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052590688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052590688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052590700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052590700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052590908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052590909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052590918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052590918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:10,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/6e2ca4ab25e6469194a28dc426cf0945 2024-11-19T21:42:10,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/0326f7b0ff104a3c8b6f3dd9bfe5e376 is 50, key is test_row_0/B:col10/1732052529374/Put/seqid=0 2024-11-19T21:42:10,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742310_1486 (size=12001) 2024-11-19T21:42:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T21:42:11,119 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:11,119 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-19T21:42:11,121 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T21:42:11,121 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:11,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:11,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052591218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052591218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T21:42:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052591229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052591229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:11,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:11,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/0326f7b0ff104a3c8b6f3dd9bfe5e376 2024-11-19T21:42:11,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/d43cead540c240ccaaeeb7b6bb8a3791 is 50, key is test_row_0/C:col10/1732052529374/Put/seqid=0 2024-11-19T21:42:11,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742311_1487 (size=12001) 2024-11-19T21:42:11,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/d43cead540c240ccaaeeb7b6bb8a3791 2024-11-19T21:42:11,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/6e2ca4ab25e6469194a28dc426cf0945 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/6e2ca4ab25e6469194a28dc426cf0945 2024-11-19T21:42:11,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/6e2ca4ab25e6469194a28dc426cf0945, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T21:42:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/0326f7b0ff104a3c8b6f3dd9bfe5e376 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0326f7b0ff104a3c8b6f3dd9bfe5e376 2024-11-19T21:42:11,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0326f7b0ff104a3c8b6f3dd9bfe5e376, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T21:42:11,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/d43cead540c240ccaaeeb7b6bb8a3791 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/d43cead540c240ccaaeeb7b6bb8a3791 2024-11-19T21:42:11,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/d43cead540c240ccaaeeb7b6bb8a3791, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T21:42:11,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for abb063485168bef9628960f3efec5fb9 in 847ms, sequenceid=90, compaction requested=true 2024-11-19T21:42:11,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:11,372 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:11,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:11,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:11,372 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:11,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:11,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:11,373 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:11,373 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/A is initiating minor compaction (all files) 2024-11-19T21:42:11,373 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:11,373 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/A in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,373 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/B is initiating minor compaction (all files) 2024-11-19T21:42:11,373 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/B in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,373 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/3a15f86c449947108422ebcdcd700c62, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/23d4bb84103a4f5fbfccb76b8c927b42, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/6e2ca4ab25e6469194a28dc426cf0945] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=35.3 K 2024-11-19T21:42:11,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:11,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:11,373 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/4dbfca20c1b041218b92f449ee5a9010, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e064f15e7af54ed7bfb2cbae2875389e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0326f7b0ff104a3c8b6f3dd9bfe5e376] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=35.3 K 2024-11-19T21:42:11,374 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dbfca20c1b041218b92f449ee5a9010, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732052526003 2024-11-19T21:42:11,374 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a15f86c449947108422ebcdcd700c62, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732052526003 2024-11-19T21:42:11,375 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23d4bb84103a4f5fbfccb76b8c927b42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732052527182 2024-11-19T21:42:11,375 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e064f15e7af54ed7bfb2cbae2875389e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732052527182 2024-11-19T21:42:11,375 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0326f7b0ff104a3c8b6f3dd9bfe5e376, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732052529370 2024-11-19T21:42:11,375 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e2ca4ab25e6469194a28dc426cf0945, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732052529370 2024-11-19T21:42:11,383 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:11,384 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#A#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:11,384 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/28f994186b9b4a8290363d99ee4eda2d is 50, key is test_row_0/A:col10/1732052529374/Put/seqid=0 2024-11-19T21:42:11,384 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/ba2bfad7678b416badabea9cc31152ad is 50, key is test_row_0/B:col10/1732052529374/Put/seqid=0 2024-11-19T21:42:11,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742312_1488 (size=12207) 2024-11-19T21:42:11,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742313_1489 (size=12207) 2024-11-19T21:42:11,400 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/ba2bfad7678b416badabea9cc31152ad as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ba2bfad7678b416badabea9cc31152ad 2024-11-19T21:42:11,406 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/B of abb063485168bef9628960f3efec5fb9 into ba2bfad7678b416badabea9cc31152ad(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:11,406 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:11,406 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/B, priority=13, startTime=1732052531372; duration=0sec 2024-11-19T21:42:11,406 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:11,406 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:B 2024-11-19T21:42:11,406 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:11,407 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:11,407 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/C is initiating minor compaction (all files) 2024-11-19T21:42:11,407 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/C in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,407 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/c96dbeb9fe29474aa7651dda7e3b1bd6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/df3da3c56f68476d8349f993cf104679, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/d43cead540c240ccaaeeb7b6bb8a3791] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=35.3 K 2024-11-19T21:42:11,408 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting c96dbeb9fe29474aa7651dda7e3b1bd6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732052526003 2024-11-19T21:42:11,408 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting df3da3c56f68476d8349f993cf104679, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732052527182 2024-11-19T21:42:11,408 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d43cead540c240ccaaeeb7b6bb8a3791, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732052529370 2024-11-19T21:42:11,414 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:11,414 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/be06636f73e94cb8bea790e118f78265 is 50, key is test_row_0/C:col10/1732052529374/Put/seqid=0 2024-11-19T21:42:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:11,416 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:11,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:42:11,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/75c675cffa704bcb8493cf1a09be45b0 is 50, key is test_row_0/A:col10/1732052530573/Put/seqid=0 2024-11-19T21:42:11,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:11,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:11,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742314_1490 (size=12207) 2024-11-19T21:42:11,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742315_1491 (size=14341) 2024-11-19T21:42:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T21:42:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052591468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:11,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:11,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052591578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052591738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052591738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052591738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052591738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T21:42:11,795 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/28f994186b9b4a8290363d99ee4eda2d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/28f994186b9b4a8290363d99ee4eda2d 2024-11-19T21:42:11,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052591788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:11,799 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/A of abb063485168bef9628960f3efec5fb9 into 28f994186b9b4a8290363d99ee4eda2d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:11,799 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:11,799 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/A, priority=13, startTime=1732052531372; duration=0sec 2024-11-19T21:42:11,799 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:11,799 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:A 2024-11-19T21:42:11,830 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/be06636f73e94cb8bea790e118f78265 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/be06636f73e94cb8bea790e118f78265 2024-11-19T21:42:11,834 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/C of abb063485168bef9628960f3efec5fb9 into be06636f73e94cb8bea790e118f78265(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:11,834 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:11,834 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/C, priority=13, startTime=1732052531372; duration=0sec 2024-11-19T21:42:11,834 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:11,834 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:C 2024-11-19T21:42:11,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/75c675cffa704bcb8493cf1a09be45b0 2024-11-19T21:42:11,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/415e592728fd455e9faac965e85710fb is 50, key is test_row_0/B:col10/1732052530573/Put/seqid=0 2024-11-19T21:42:11,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742316_1492 (size=12001) 2024-11-19T21:42:11,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:11,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:11,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:11,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:12,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:12,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052592108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/415e592728fd455e9faac965e85710fb 2024-11-19T21:42:12,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/675191db32cf4364845220ec801edc3c is 50, key is test_row_0/C:col10/1732052530573/Put/seqid=0 2024-11-19T21:42:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T21:42:12,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742317_1493 (size=12001) 2024-11-19T21:42:12,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:12,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:12,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:12,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052592618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/675191db32cf4364845220ec801edc3c 2024-11-19T21:42:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/75c675cffa704bcb8493cf1a09be45b0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/75c675cffa704bcb8493cf1a09be45b0 2024-11-19T21:42:12,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/75c675cffa704bcb8493cf1a09be45b0, entries=200, sequenceid=115, filesize=14.0 K 2024-11-19T21:42:12,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/415e592728fd455e9faac965e85710fb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/415e592728fd455e9faac965e85710fb 2024-11-19T21:42:12,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/415e592728fd455e9faac965e85710fb, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T21:42:12,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/675191db32cf4364845220ec801edc3c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/675191db32cf4364845220ec801edc3c 2024-11-19T21:42:12,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/675191db32cf4364845220ec801edc3c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T21:42:12,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for abb063485168bef9628960f3efec5fb9 in 1262ms, sequenceid=115, compaction requested=false 2024-11-19T21:42:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:12,751 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:12,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:42:12,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/8325ccb58e5d4b33b1c18fc45763ced4 is 50, key is test_row_0/A:col10/1732052531457/Put/seqid=0 2024-11-19T21:42:12,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742318_1494 (size=14391) 2024-11-19T21:42:12,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052592792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052592792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052592793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052592794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052592908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052592908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052592908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:12,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052592908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:12,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:12,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:13,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:13,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052593118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052593118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052593118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052593118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/8325ccb58e5d4b33b1c18fc45763ced4 2024-11-19T21:42:13,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/df9cad67b00345ac882d7fd68eef7ab4 is 50, key is test_row_0/B:col10/1732052531457/Put/seqid=0 2024-11-19T21:42:13,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742319_1495 (size=12051) 2024-11-19T21:42:13,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:13,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:13,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T21:42:13,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052593427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052593428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052593428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052593428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:13,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:13,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/df9cad67b00345ac882d7fd68eef7ab4 2024-11-19T21:42:13,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/b2c94f96676b4c99a6a32092d5b96ffb is 50, key is test_row_0/C:col10/1732052531457/Put/seqid=0 2024-11-19T21:42:13,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742320_1496 (size=12051) 2024-11-19T21:42:13,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T21:42:13,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T21:42:13,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052593638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,645 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in abb063485168bef9628960f3efec5fb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:13,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:13,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:13,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:13,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:13,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052593938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052593938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052593938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052593939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:13,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/b2c94f96676b4c99a6a32092d5b96ffb 2024-11-19T21:42:13,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/8325ccb58e5d4b33b1c18fc45763ced4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8325ccb58e5d4b33b1c18fc45763ced4 2024-11-19T21:42:13,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8325ccb58e5d4b33b1c18fc45763ced4, entries=200, sequenceid=130, filesize=14.1 K 2024-11-19T21:42:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/df9cad67b00345ac882d7fd68eef7ab4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/df9cad67b00345ac882d7fd68eef7ab4 2024-11-19T21:42:13,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/df9cad67b00345ac882d7fd68eef7ab4, entries=150, sequenceid=130, filesize=11.8 K 2024-11-19T21:42:13,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/b2c94f96676b4c99a6a32092d5b96ffb as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b2c94f96676b4c99a6a32092d5b96ffb 2024-11-19T21:42:13,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b2c94f96676b4c99a6a32092d5b96ffb, entries=150, sequenceid=130, filesize=11.8 K 2024-11-19T21:42:13,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for abb063485168bef9628960f3efec5fb9 in 1240ms, sequenceid=130, compaction requested=true 2024-11-19T21:42:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:13,991 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:13,991 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:13,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:13,992 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40939 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:13,992 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:13,992 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/A is initiating minor compaction (all files) 2024-11-19T21:42:13,992 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/B is initiating minor compaction (all files) 2024-11-19T21:42:13,992 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/B in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,992 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/A in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:13,992 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/28f994186b9b4a8290363d99ee4eda2d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/75c675cffa704bcb8493cf1a09be45b0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8325ccb58e5d4b33b1c18fc45763ced4] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=40.0 K 2024-11-19T21:42:13,992 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ba2bfad7678b416badabea9cc31152ad, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/415e592728fd455e9faac965e85710fb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/df9cad67b00345ac882d7fd68eef7ab4] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=35.4 K 2024-11-19T21:42:13,993 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28f994186b9b4a8290363d99ee4eda2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732052529370 2024-11-19T21:42:13,993 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting ba2bfad7678b416badabea9cc31152ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732052529370 2024-11-19T21:42:13,993 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c675cffa704bcb8493cf1a09be45b0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732052530562 2024-11-19T21:42:13,993 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8325ccb58e5d4b33b1c18fc45763ced4, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732052531453 2024-11-19T21:42:13,994 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 415e592728fd455e9faac965e85710fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732052530572 2024-11-19T21:42:13,994 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting df9cad67b00345ac882d7fd68eef7ab4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732052531457 2024-11-19T21:42:13,998 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:13,999 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/6def325aed024d7fbab4a8d9bf965e8b is 50, key is test_row_0/B:col10/1732052531457/Put/seqid=0 2024-11-19T21:42:13,999 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#A#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:13,999 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/4245f5b8ac204fdba9726c8ed3110fb1 is 50, key is test_row_0/A:col10/1732052531457/Put/seqid=0 2024-11-19T21:42:14,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742321_1497 (size=12359) 2024-11-19T21:42:14,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742322_1498 (size=12359) 2024-11-19T21:42:14,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T21:42:14,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:14,019 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:42:14,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/cbe1d90fd7c7464b9cda7dd48d795eb6 is 50, key is test_row_0/A:col10/1732052532781/Put/seqid=0 2024-11-19T21:42:14,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742323_1499 (size=12151) 2024-11-19T21:42:14,409 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/6def325aed024d7fbab4a8d9bf965e8b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6def325aed024d7fbab4a8d9bf965e8b 2024-11-19T21:42:14,410 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/4245f5b8ac204fdba9726c8ed3110fb1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4245f5b8ac204fdba9726c8ed3110fb1 2024-11-19T21:42:14,413 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/B of abb063485168bef9628960f3efec5fb9 into 6def325aed024d7fbab4a8d9bf965e8b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:14,413 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/A of abb063485168bef9628960f3efec5fb9 into 4245f5b8ac204fdba9726c8ed3110fb1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:14,414 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:14,414 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:14,414 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/B, priority=13, startTime=1732052533991; duration=0sec 2024-11-19T21:42:14,414 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/A, priority=13, startTime=1732052533991; duration=0sec 2024-11-19T21:42:14,414 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:14,414 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:B 2024-11-19T21:42:14,414 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:14,414 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:A 2024-11-19T21:42:14,414 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:14,414 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:14,415 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/C is initiating minor compaction (all files) 2024-11-19T21:42:14,415 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/C in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:14,415 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/be06636f73e94cb8bea790e118f78265, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/675191db32cf4364845220ec801edc3c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b2c94f96676b4c99a6a32092d5b96ffb] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=35.4 K 2024-11-19T21:42:14,415 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting be06636f73e94cb8bea790e118f78265, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732052529370 2024-11-19T21:42:14,415 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 675191db32cf4364845220ec801edc3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732052530572 2024-11-19T21:42:14,415 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b2c94f96676b4c99a6a32092d5b96ffb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732052531457 2024-11-19T21:42:14,420 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:14,420 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/1882c4f12453478e92005206d2eec5d4 is 50, key is test_row_0/C:col10/1732052531457/Put/seqid=0 2024-11-19T21:42:14,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742324_1500 (size=12359) 2024-11-19T21:42:14,429 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/cbe1d90fd7c7464b9cda7dd48d795eb6 2024-11-19T21:42:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/11597c218fc3476cb2e772828011308c is 50, key is test_row_0/B:col10/1732052532781/Put/seqid=0 2024-11-19T21:42:14,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742325_1501 (size=12151) 2024-11-19T21:42:14,826 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/1882c4f12453478e92005206d2eec5d4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1882c4f12453478e92005206d2eec5d4 2024-11-19T21:42:14,829 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/C of abb063485168bef9628960f3efec5fb9 into 1882c4f12453478e92005206d2eec5d4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:14,829 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:14,829 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/C, priority=13, startTime=1732052533991; duration=0sec 2024-11-19T21:42:14,829 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:14,829 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:C 2024-11-19T21:42:14,838 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/11597c218fc3476cb2e772828011308c 2024-11-19T21:42:14,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/e29398bf9cc3412fa2474619b2dd6e42 is 50, key is test_row_0/C:col10/1732052532781/Put/seqid=0 2024-11-19T21:42:14,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742326_1502 (size=12151) 2024-11-19T21:42:14,848 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/e29398bf9cc3412fa2474619b2dd6e42 2024-11-19T21:42:14,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/cbe1d90fd7c7464b9cda7dd48d795eb6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cbe1d90fd7c7464b9cda7dd48d795eb6 2024-11-19T21:42:14,860 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cbe1d90fd7c7464b9cda7dd48d795eb6, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T21:42:14,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/11597c218fc3476cb2e772828011308c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/11597c218fc3476cb2e772828011308c 2024-11-19T21:42:14,863 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/11597c218fc3476cb2e772828011308c, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T21:42:14,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/e29398bf9cc3412fa2474619b2dd6e42 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e29398bf9cc3412fa2474619b2dd6e42 2024-11-19T21:42:14,866 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e29398bf9cc3412fa2474619b2dd6e42, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T21:42:14,867 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for abb063485168bef9628960f3efec5fb9 in 849ms, sequenceid=153, compaction requested=false 2024-11-19T21:42:14,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:14,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:14,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-19T21:42:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-19T21:42:14,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T21:42:14,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7480 sec 2024-11-19T21:42:14,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 3.7520 sec 2024-11-19T21:42:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:14,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:14,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:42:14,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/4fdc8fb942ea4a8eae9b31e220561c30 is 50, key is test_row_0/A:col10/1732052534956/Put/seqid=0 2024-11-19T21:42:14,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742327_1503 (size=24105) 2024-11-19T21:42:14,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/4fdc8fb942ea4a8eae9b31e220561c30 2024-11-19T21:42:14,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/b9960b50dd404b1098540193a1a6c4b0 is 50, key is test_row_0/B:col10/1732052534956/Put/seqid=0 2024-11-19T21:42:14,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742328_1504 (size=12151) 2024-11-19T21:42:15,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052594994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052594995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052594995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052594998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052595107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052595108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052595108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052595108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T21:42:15,279 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:15,279 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-19T21:42:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T21:42:15,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:15,281 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:15,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052595318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052595318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052595318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052595319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/b9960b50dd404b1098540193a1a6c4b0 2024-11-19T21:42:15,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/ff1fe2bf08474daf93baae62c0c11df5 is 50, key is test_row_0/C:col10/1732052534956/Put/seqid=0 2024-11-19T21:42:15,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742329_1505 (size=12151) 2024-11-19T21:42:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T21:42:15,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T21:42:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:15,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:15,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T21:42:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:15,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T21:42:15,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052595628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052595628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052595628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052595628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052595658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:15,665 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in abb063485168bef9628960f3efec5fb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:15,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T21:42:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:15,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:15,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/ff1fe2bf08474daf93baae62c0c11df5 2024-11-19T21:42:15,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/4fdc8fb942ea4a8eae9b31e220561c30 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4fdc8fb942ea4a8eae9b31e220561c30 2024-11-19T21:42:15,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4fdc8fb942ea4a8eae9b31e220561c30, entries=400, sequenceid=168, filesize=23.5 K 2024-11-19T21:42:15,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/b9960b50dd404b1098540193a1a6c4b0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/b9960b50dd404b1098540193a1a6c4b0 2024-11-19T21:42:15,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/b9960b50dd404b1098540193a1a6c4b0, entries=150, sequenceid=168, filesize=11.9 K 2024-11-19T21:42:15,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/ff1fe2bf08474daf93baae62c0c11df5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/ff1fe2bf08474daf93baae62c0c11df5 2024-11-19T21:42:15,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/ff1fe2bf08474daf93baae62c0c11df5, entries=150, sequenceid=168, filesize=11.9 K 2024-11-19T21:42:15,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for abb063485168bef9628960f3efec5fb9 in 842ms, sequenceid=168, compaction requested=true 2024-11-19T21:42:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:15,798 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:15,798 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:15,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:15,799 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:15,799 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 48615 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:15,799 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/B is initiating minor compaction (all files) 2024-11-19T21:42:15,799 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/A is initiating minor compaction (all files) 2024-11-19T21:42:15,799 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/A in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:15,799 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/B in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:15,799 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4245f5b8ac204fdba9726c8ed3110fb1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cbe1d90fd7c7464b9cda7dd48d795eb6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4fdc8fb942ea4a8eae9b31e220561c30] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=47.5 K 2024-11-19T21:42:15,799 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6def325aed024d7fbab4a8d9bf965e8b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/11597c218fc3476cb2e772828011308c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/b9960b50dd404b1098540193a1a6c4b0] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=35.8 K 2024-11-19T21:42:15,799 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6def325aed024d7fbab4a8d9bf965e8b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732052531457 2024-11-19T21:42:15,799 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4245f5b8ac204fdba9726c8ed3110fb1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732052531457 2024-11-19T21:42:15,800 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 11597c218fc3476cb2e772828011308c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732052532781 2024-11-19T21:42:15,800 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbe1d90fd7c7464b9cda7dd48d795eb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732052532781 2024-11-19T21:42:15,800 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b9960b50dd404b1098540193a1a6c4b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732052534954 2024-11-19T21:42:15,800 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fdc8fb942ea4a8eae9b31e220561c30, keycount=400, bloomtype=ROW, size=23.5 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732052534948 2024-11-19T21:42:15,805 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#A#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:15,805 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/9e7e54366e4c4a8a9ee9024d123f36bd is 50, key is test_row_0/A:col10/1732052534956/Put/seqid=0 2024-11-19T21:42:15,807 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:15,808 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/6269160f2daf4291aa00e1d0eae68d7e is 50, key is test_row_0/B:col10/1732052534956/Put/seqid=0 2024-11-19T21:42:15,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742330_1506 (size=12561) 2024-11-19T21:42:15,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742331_1507 (size=12561) 2024-11-19T21:42:15,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T21:42:15,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:15,891 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:42:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/a838a4c10fbb488b8f4a0c4b10f10a3d is 50, key is test_row_0/A:col10/1732052534981/Put/seqid=0 2024-11-19T21:42:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742332_1508 (size=12151) 2024-11-19T21:42:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T21:42:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:16,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052596155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052596156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052596159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052596162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,213 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/9e7e54366e4c4a8a9ee9024d123f36bd as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/9e7e54366e4c4a8a9ee9024d123f36bd 2024-11-19T21:42:16,214 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/6269160f2daf4291aa00e1d0eae68d7e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6269160f2daf4291aa00e1d0eae68d7e 2024-11-19T21:42:16,217 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/B of abb063485168bef9628960f3efec5fb9 into 6269160f2daf4291aa00e1d0eae68d7e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:16,217 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/A of abb063485168bef9628960f3efec5fb9 into 9e7e54366e4c4a8a9ee9024d123f36bd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:16,217 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:16,217 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:16,217 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/B, priority=13, startTime=1732052535798; duration=0sec 2024-11-19T21:42:16,217 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/A, priority=13, startTime=1732052535798; duration=0sec 2024-11-19T21:42:16,217 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:16,217 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:B 2024-11-19T21:42:16,217 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:16,217 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:A 2024-11-19T21:42:16,217 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:16,217 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:16,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/C is initiating minor compaction (all files) 2024-11-19T21:42:16,218 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/C in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:16,218 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1882c4f12453478e92005206d2eec5d4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e29398bf9cc3412fa2474619b2dd6e42, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/ff1fe2bf08474daf93baae62c0c11df5] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=35.8 K 2024-11-19T21:42:16,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 1882c4f12453478e92005206d2eec5d4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732052531457 2024-11-19T21:42:16,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e29398bf9cc3412fa2474619b2dd6e42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732052532781 2024-11-19T21:42:16,218 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1fe2bf08474daf93baae62c0c11df5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732052534954 2024-11-19T21:42:16,224 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#C#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:16,224 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/2a90f28d9ba446559a1e0452c403ab03 is 50, key is test_row_0/C:col10/1732052534956/Put/seqid=0 2024-11-19T21:42:16,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742333_1509 (size=12561) 2024-11-19T21:42:16,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052596268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052596268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052596268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052596278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,298 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/a838a4c10fbb488b8f4a0c4b10f10a3d 2024-11-19T21:42:16,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/885d7ee7294c4309b9fe3cdd36104278 is 50, key is test_row_0/B:col10/1732052534981/Put/seqid=0 2024-11-19T21:42:16,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742334_1510 (size=12151) 2024-11-19T21:42:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T21:42:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052596478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052596478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052596478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052596488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,631 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/2a90f28d9ba446559a1e0452c403ab03 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/2a90f28d9ba446559a1e0452c403ab03 2024-11-19T21:42:16,634 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/C of abb063485168bef9628960f3efec5fb9 into 2a90f28d9ba446559a1e0452c403ab03(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:16,634 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:16,634 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/C, priority=13, startTime=1732052535798; duration=0sec 2024-11-19T21:42:16,634 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:16,634 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:C 2024-11-19T21:42:16,707 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/885d7ee7294c4309b9fe3cdd36104278 2024-11-19T21:42:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/43a929b70db6414b8bb8c31b46fd9bed is 50, key is test_row_0/C:col10/1732052534981/Put/seqid=0 2024-11-19T21:42:16,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742335_1511 (size=12151) 2024-11-19T21:42:16,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052596788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052596788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052596788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:16,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052596798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,118 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/43a929b70db6414b8bb8c31b46fd9bed 2024-11-19T21:42:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/a838a4c10fbb488b8f4a0c4b10f10a3d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a838a4c10fbb488b8f4a0c4b10f10a3d 2024-11-19T21:42:17,171 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a838a4c10fbb488b8f4a0c4b10f10a3d, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T21:42:17,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/885d7ee7294c4309b9fe3cdd36104278 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/885d7ee7294c4309b9fe3cdd36104278 2024-11-19T21:42:17,174 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/885d7ee7294c4309b9fe3cdd36104278, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T21:42:17,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/43a929b70db6414b8bb8c31b46fd9bed as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/43a929b70db6414b8bb8c31b46fd9bed 2024-11-19T21:42:17,178 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/43a929b70db6414b8bb8c31b46fd9bed, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T21:42:17,179 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for abb063485168bef9628960f3efec5fb9 in 1287ms, sequenceid=192, compaction requested=false 2024-11-19T21:42:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-19T21:42:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-19T21:42:17,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T21:42:17,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8990 sec 2024-11-19T21:42:17,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9030 sec 2024-11-19T21:42:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:17,302 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:17,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:42:17,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/866644b8752a470d97b6e296ef0dde22 is 50, key is test_row_0/A:col10/1732052536158/Put/seqid=0 2024-11-19T21:42:17,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742336_1512 (size=14541) 2024-11-19T21:42:17,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052597347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052597353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052597354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052597357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T21:42:17,428 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:17,428 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T21:42:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T21:42:17,430 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:17,431 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:17,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052597468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052597468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052597468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052597469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T21:42:17,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T21:42:17,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:17,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:17,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:17,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:17,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052597678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052597678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052597688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052597688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:17,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/866644b8752a470d97b6e296ef0dde22 2024-11-19T21:42:17,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/bfb2c7b2aac042e68cbba4c40ac6ea0c is 50, key is test_row_0/B:col10/1732052536158/Put/seqid=0 2024-11-19T21:42:17,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742337_1513 (size=12151) 2024-11-19T21:42:17,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T21:42:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:17,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T21:42:17,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T21:42:17,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:17,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:17,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:17,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052597988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052597998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052597998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052597998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T21:42:18,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:18,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:18,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:18,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T21:42:18,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/bfb2c7b2aac042e68cbba4c40ac6ea0c 2024-11-19T21:42:18,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/1e48b34cf6394ab79057d8752b2d22b7 is 50, key is test_row_0/C:col10/1732052536158/Put/seqid=0 2024-11-19T21:42:18,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742338_1514 (size=12151) 2024-11-19T21:42:18,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/1e48b34cf6394ab79057d8752b2d22b7 2024-11-19T21:42:18,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/866644b8752a470d97b6e296ef0dde22 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/866644b8752a470d97b6e296ef0dde22 2024-11-19T21:42:18,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/866644b8752a470d97b6e296ef0dde22, entries=200, sequenceid=208, filesize=14.2 K 2024-11-19T21:42:18,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/bfb2c7b2aac042e68cbba4c40ac6ea0c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/bfb2c7b2aac042e68cbba4c40ac6ea0c 2024-11-19T21:42:18,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/bfb2c7b2aac042e68cbba4c40ac6ea0c, entries=150, sequenceid=208, filesize=11.9 K 2024-11-19T21:42:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/1e48b34cf6394ab79057d8752b2d22b7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1e48b34cf6394ab79057d8752b2d22b7 2024-11-19T21:42:18,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1e48b34cf6394ab79057d8752b2d22b7, entries=150, sequenceid=208, filesize=11.9 K 2024-11-19T21:42:18,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for abb063485168bef9628960f3efec5fb9 in 868ms, sequenceid=208, compaction requested=true 2024-11-19T21:42:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:18,170 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:18,170 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:18,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:18,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:18,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:18,171 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:18,171 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:18,171 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/A is initiating minor compaction (all files) 2024-11-19T21:42:18,171 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/B is initiating minor compaction (all files) 2024-11-19T21:42:18,171 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/B in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:18,171 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/A in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:18,171 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/9e7e54366e4c4a8a9ee9024d123f36bd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a838a4c10fbb488b8f4a0c4b10f10a3d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/866644b8752a470d97b6e296ef0dde22] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=38.3 K 2024-11-19T21:42:18,171 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6269160f2daf4291aa00e1d0eae68d7e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/885d7ee7294c4309b9fe3cdd36104278, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/bfb2c7b2aac042e68cbba4c40ac6ea0c] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=36.0 K 2024-11-19T21:42:18,172 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6269160f2daf4291aa00e1d0eae68d7e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732052534954 2024-11-19T21:42:18,172 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e7e54366e4c4a8a9ee9024d123f36bd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732052534954 2024-11-19T21:42:18,172 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting a838a4c10fbb488b8f4a0c4b10f10a3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732052534981 2024-11-19T21:42:18,172 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 885d7ee7294c4309b9fe3cdd36104278, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732052534981 2024-11-19T21:42:18,172 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 866644b8752a470d97b6e296ef0dde22, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732052536144 2024-11-19T21:42:18,172 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb2c7b2aac042e68cbba4c40ac6ea0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732052536144 2024-11-19T21:42:18,177 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:18,178 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/2c42262bd77841b2a30f063348fbf8ce is 50, key is test_row_0/B:col10/1732052536158/Put/seqid=0 2024-11-19T21:42:18,180 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#A#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:18,181 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/cacf5adb70484043adf5b091fa2a1487 is 50, key is test_row_0/A:col10/1732052536158/Put/seqid=0 2024-11-19T21:42:18,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742340_1516 (size=12663) 2024-11-19T21:42:18,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T21:42:18,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:18,193 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:42:18,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/167bfe0f6f7740bca1727784a04c3f52 is 50, key is test_row_0/A:col10/1732052537342/Put/seqid=0 2024-11-19T21:42:18,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742339_1515 (size=12663) 2024-11-19T21:42:18,208 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/2c42262bd77841b2a30f063348fbf8ce as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/2c42262bd77841b2a30f063348fbf8ce 2024-11-19T21:42:18,214 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/B of abb063485168bef9628960f3efec5fb9 into 2c42262bd77841b2a30f063348fbf8ce(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:18,214 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:18,214 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/B, priority=13, startTime=1732052538170; duration=0sec 2024-11-19T21:42:18,214 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:18,214 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:B 2024-11-19T21:42:18,214 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:18,216 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:18,216 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/C is initiating minor compaction (all files) 2024-11-19T21:42:18,216 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/C in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:18,216 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/2a90f28d9ba446559a1e0452c403ab03, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/43a929b70db6414b8bb8c31b46fd9bed, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1e48b34cf6394ab79057d8752b2d22b7] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=36.0 K 2024-11-19T21:42:18,217 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a90f28d9ba446559a1e0452c403ab03, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732052534954 2024-11-19T21:42:18,217 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a929b70db6414b8bb8c31b46fd9bed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732052534981 2024-11-19T21:42:18,217 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e48b34cf6394ab79057d8752b2d22b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732052536144 2024-11-19T21:42:18,224 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:18,225 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/8f1e9610a71549789596892c9f3d415b is 50, key is test_row_0/C:col10/1732052536158/Put/seqid=0 2024-11-19T21:42:18,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742341_1517 (size=12151) 2024-11-19T21:42:18,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742342_1518 (size=12663) 2024-11-19T21:42:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:18,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:18,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052598534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052598535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052598536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052598540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T21:42:18,594 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/cacf5adb70484043adf5b091fa2a1487 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cacf5adb70484043adf5b091fa2a1487 2024-11-19T21:42:18,598 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/A of abb063485168bef9628960f3efec5fb9 into cacf5adb70484043adf5b091fa2a1487(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:18,598 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:18,598 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/A, priority=13, startTime=1732052538170; duration=0sec 2024-11-19T21:42:18,599 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:18,599 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:A 2024-11-19T21:42:18,629 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/167bfe0f6f7740bca1727784a04c3f52 2024-11-19T21:42:18,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/5831119008ef4870913dd6ce2186cc42 is 50, key is test_row_0/B:col10/1732052537342/Put/seqid=0 2024-11-19T21:42:18,644 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/8f1e9610a71549789596892c9f3d415b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/8f1e9610a71549789596892c9f3d415b 2024-11-19T21:42:18,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742343_1519 (size=12151) 2024-11-19T21:42:18,645 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/5831119008ef4870913dd6ce2186cc42 2024-11-19T21:42:18,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,652 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/C of abb063485168bef9628960f3efec5fb9 into 8f1e9610a71549789596892c9f3d415b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052598649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,652 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:18,652 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/C, priority=13, startTime=1732052538171; duration=0sec 2024-11-19T21:42:18,652 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:18,652 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:C 2024-11-19T21:42:18,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052598649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052598649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052598650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/f8e3cbb471cb4494ba68b5b2dd59f44d is 50, key is test_row_0/C:col10/1732052537342/Put/seqid=0 2024-11-19T21:42:18,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742344_1520 (size=12151) 2024-11-19T21:42:18,664 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/f8e3cbb471cb4494ba68b5b2dd59f44d 2024-11-19T21:42:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/167bfe0f6f7740bca1727784a04c3f52 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/167bfe0f6f7740bca1727784a04c3f52 2024-11-19T21:42:18,675 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/167bfe0f6f7740bca1727784a04c3f52, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T21:42:18,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/5831119008ef4870913dd6ce2186cc42 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/5831119008ef4870913dd6ce2186cc42 2024-11-19T21:42:18,681 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/5831119008ef4870913dd6ce2186cc42, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T21:42:18,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/f8e3cbb471cb4494ba68b5b2dd59f44d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/f8e3cbb471cb4494ba68b5b2dd59f44d 2024-11-19T21:42:18,686 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/f8e3cbb471cb4494ba68b5b2dd59f44d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T21:42:18,687 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for abb063485168bef9628960f3efec5fb9 in 494ms, sequenceid=231, compaction requested=false 2024-11-19T21:42:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T21:42:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T21:42:18,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T21:42:18,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2580 sec 2024-11-19T21:42:18,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.2620 sec 2024-11-19T21:42:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:18,860 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:18,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T21:42:18,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/b39af114c0d6456a82718ab0c9109817 is 50, key is test_row_0/A:col10/1732052538524/Put/seqid=0 2024-11-19T21:42:18,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742345_1521 (size=14541) 2024-11-19T21:42:18,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/b39af114c0d6456a82718ab0c9109817 2024-11-19T21:42:18,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/61f271cc772b441d98fde7c50db7688d is 50, key is test_row_0/B:col10/1732052538524/Put/seqid=0 2024-11-19T21:42:18,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742346_1522 (size=12151) 2024-11-19T21:42:18,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052598895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052598896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052598896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052598897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:18,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:18,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:18,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052599008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052599008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052599008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052599008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052599218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052599218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052599218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052599218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/61f271cc772b441d98fde7c50db7688d 2024-11-19T21:42:19,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/001a8ac3ae4546a9b43c08e6ae372e92 is 50, key is test_row_0/C:col10/1732052538524/Put/seqid=0 2024-11-19T21:42:19,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742347_1523 (size=12151) 2024-11-19T21:42:19,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052599528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052599528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052599528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052599528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T21:42:19,579 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:19,579 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T21:42:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T21:42:19,581 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:19,581 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:19,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:19,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34140 deadline: 1732052599678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:19,683 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in abb063485168bef9628960f3efec5fb9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:19,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:19,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:19,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/001a8ac3ae4546a9b43c08e6ae372e92 2024-11-19T21:42:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T21:42:19,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/b39af114c0d6456a82718ab0c9109817 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b39af114c0d6456a82718ab0c9109817 2024-11-19T21:42:19,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b39af114c0d6456a82718ab0c9109817, entries=200, sequenceid=248, filesize=14.2 K 2024-11-19T21:42:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/61f271cc772b441d98fde7c50db7688d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/61f271cc772b441d98fde7c50db7688d 2024-11-19T21:42:19,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/61f271cc772b441d98fde7c50db7688d, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T21:42:19,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/001a8ac3ae4546a9b43c08e6ae372e92 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/001a8ac3ae4546a9b43c08e6ae372e92 2024-11-19T21:42:19,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/001a8ac3ae4546a9b43c08e6ae372e92, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T21:42:19,699 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T21:42:19,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for abb063485168bef9628960f3efec5fb9 in 840ms, sequenceid=248, compaction requested=true 2024-11-19T21:42:19,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:19,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:19,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:19,700 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:19,700 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:19,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:19,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:19,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:19,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:19,701 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:19,701 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/A is initiating minor compaction (all files) 2024-11-19T21:42:19,701 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/A in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:19,701 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cacf5adb70484043adf5b091fa2a1487, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/167bfe0f6f7740bca1727784a04c3f52, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b39af114c0d6456a82718ab0c9109817] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=38.4 K 2024-11-19T21:42:19,701 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:19,702 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/B is initiating minor compaction (all files) 2024-11-19T21:42:19,702 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/B in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:19,702 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/2c42262bd77841b2a30f063348fbf8ce, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/5831119008ef4870913dd6ce2186cc42, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/61f271cc772b441d98fde7c50db7688d] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=36.1 K 2024-11-19T21:42:19,702 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c42262bd77841b2a30f063348fbf8ce, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732052536144 2024-11-19T21:42:19,702 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting cacf5adb70484043adf5b091fa2a1487, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732052536144 2024-11-19T21:42:19,702 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 167bfe0f6f7740bca1727784a04c3f52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732052537342 2024-11-19T21:42:19,702 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 5831119008ef4870913dd6ce2186cc42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732052537342 2024-11-19T21:42:19,702 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting b39af114c0d6456a82718ab0c9109817, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732052538524 2024-11-19T21:42:19,703 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 61f271cc772b441d98fde7c50db7688d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732052538524 2024-11-19T21:42:19,707 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:19,708 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/39b3186d35084974992567866c0dbfbd is 50, key is test_row_0/B:col10/1732052538524/Put/seqid=0 2024-11-19T21:42:19,708 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#A#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:19,708 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/a78fd35cf9c743969d885e888bde8290 is 50, key is test_row_0/A:col10/1732052538524/Put/seqid=0 2024-11-19T21:42:19,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742349_1525 (size=12765) 2024-11-19T21:42:19,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742348_1524 (size=12765) 2024-11-19T21:42:19,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T21:42:19,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:19,733 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T21:42:19,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/13f6eaa27e19453093dd2c7c655ce542 is 50, key is test_row_0/A:col10/1732052538885/Put/seqid=0 2024-11-19T21:42:19,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742350_1526 (size=12301) 2024-11-19T21:42:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T21:42:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:20,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052600081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052600081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052600081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052600082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,123 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/a78fd35cf9c743969d885e888bde8290 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a78fd35cf9c743969d885e888bde8290 2024-11-19T21:42:20,124 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/39b3186d35084974992567866c0dbfbd as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/39b3186d35084974992567866c0dbfbd 2024-11-19T21:42:20,130 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/B of abb063485168bef9628960f3efec5fb9 into 39b3186d35084974992567866c0dbfbd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:20,130 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/A of abb063485168bef9628960f3efec5fb9 into a78fd35cf9c743969d885e888bde8290(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:20,130 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:20,130 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/B, priority=13, startTime=1732052539700; duration=0sec 2024-11-19T21:42:20,130 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:20,130 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/A, priority=13, startTime=1732052539700; duration=0sec 2024-11-19T21:42:20,130 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:20,130 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:20,130 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:B 2024-11-19T21:42:20,130 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:A 2024-11-19T21:42:20,130 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:20,131 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:20,131 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/C is initiating minor compaction (all files) 2024-11-19T21:42:20,131 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/C in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:20,131 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/8f1e9610a71549789596892c9f3d415b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/f8e3cbb471cb4494ba68b5b2dd59f44d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/001a8ac3ae4546a9b43c08e6ae372e92] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=36.1 K 2024-11-19T21:42:20,131 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f1e9610a71549789596892c9f3d415b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732052536144 2024-11-19T21:42:20,132 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e3cbb471cb4494ba68b5b2dd59f44d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732052537342 2024-11-19T21:42:20,132 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 001a8ac3ae4546a9b43c08e6ae372e92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732052538524 2024-11-19T21:42:20,138 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:20,138 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/57ef3614b2c94a43bca8700845caa891 is 50, key is test_row_0/C:col10/1732052538524/Put/seqid=0 2024-11-19T21:42:20,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742351_1527 (size=12765) 2024-11-19T21:42:20,149 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/57ef3614b2c94a43bca8700845caa891 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/57ef3614b2c94a43bca8700845caa891 2024-11-19T21:42:20,153 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/C of abb063485168bef9628960f3efec5fb9 into 57ef3614b2c94a43bca8700845caa891(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:20,153 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:20,153 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/C, priority=13, startTime=1732052539701; duration=0sec 2024-11-19T21:42:20,153 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:20,154 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:C 2024-11-19T21:42:20,156 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/13f6eaa27e19453093dd2c7c655ce542 2024-11-19T21:42:20,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/0f0f795e4c0a488da464126ea3d0dae8 is 50, key is test_row_0/B:col10/1732052538885/Put/seqid=0 2024-11-19T21:42:20,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742352_1528 (size=12301) 2024-11-19T21:42:20,165 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/0f0f795e4c0a488da464126ea3d0dae8 2024-11-19T21:42:20,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/e8b84a7d9bdf444eb32e8aa52c2e048c is 50, key is test_row_0/C:col10/1732052538885/Put/seqid=0 2024-11-19T21:42:20,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742353_1529 (size=12301) 2024-11-19T21:42:20,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052600198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052600198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052600198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052600198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T21:42:20,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052600408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052600408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052600418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052600418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,573 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/e8b84a7d9bdf444eb32e8aa52c2e048c 2024-11-19T21:42:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/13f6eaa27e19453093dd2c7c655ce542 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/13f6eaa27e19453093dd2c7c655ce542 2024-11-19T21:42:20,579 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/13f6eaa27e19453093dd2c7c655ce542, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T21:42:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/0f0f795e4c0a488da464126ea3d0dae8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0f0f795e4c0a488da464126ea3d0dae8 2024-11-19T21:42:20,582 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0f0f795e4c0a488da464126ea3d0dae8, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T21:42:20,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/e8b84a7d9bdf444eb32e8aa52c2e048c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e8b84a7d9bdf444eb32e8aa52c2e048c 2024-11-19T21:42:20,586 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e8b84a7d9bdf444eb32e8aa52c2e048c, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T21:42:20,586 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for abb063485168bef9628960f3efec5fb9 in 853ms, sequenceid=270, compaction requested=false 2024-11-19T21:42:20,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:20,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:20,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T21:42:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T21:42:20,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T21:42:20,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0070 sec 2024-11-19T21:42:20,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0100 sec 2024-11-19T21:42:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T21:42:20,719 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:20,719 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-19T21:42:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T21:42:20,721 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:20,721 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:20,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:20,723 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:20,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T21:42:20,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/5270c790c441450494984808300b6a9d is 50, key is test_row_0/A:col10/1732052540071/Put/seqid=0 2024-11-19T21:42:20,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742354_1530 (size=14741) 2024-11-19T21:42:20,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052600756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052600757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052600759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052600760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T21:42:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052600868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052600868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T21:42:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:20,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:20,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052600878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052600878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T21:42:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T21:42:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052601078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052601078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052601088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052601088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/5270c790c441450494984808300b6a9d 2024-11-19T21:42:21,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/768538c4d4734424aa2019124732adaf is 50, key is test_row_0/B:col10/1732052540071/Put/seqid=0 2024-11-19T21:42:21,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742355_1531 (size=12301) 2024-11-19T21:42:21,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T21:42:21,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:21,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T21:42:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:21,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T21:42:21,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052601388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052601388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052601397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052601408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T21:42:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/768538c4d4734424aa2019124732adaf 2024-11-19T21:42:21,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/535acc57e97d4ec7a283e17369e6c00f is 50, key is test_row_0/C:col10/1732052540071/Put/seqid=0 2024-11-19T21:42:21,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742356_1532 (size=12301) 2024-11-19T21:42:21,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T21:42:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T21:42:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T21:42:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052601898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052601898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052601907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052601918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:21,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:21,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T21:42:21,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:21,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/535acc57e97d4ec7a283e17369e6c00f 2024-11-19T21:42:21,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/5270c790c441450494984808300b6a9d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/5270c790c441450494984808300b6a9d 2024-11-19T21:42:21,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/5270c790c441450494984808300b6a9d, entries=200, sequenceid=289, filesize=14.4 K 2024-11-19T21:42:21,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/768538c4d4734424aa2019124732adaf as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/768538c4d4734424aa2019124732adaf 2024-11-19T21:42:21,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/768538c4d4734424aa2019124732adaf, entries=150, sequenceid=289, filesize=12.0 K 2024-11-19T21:42:21,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/535acc57e97d4ec7a283e17369e6c00f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/535acc57e97d4ec7a283e17369e6c00f 2024-11-19T21:42:21,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/535acc57e97d4ec7a283e17369e6c00f, entries=150, sequenceid=289, filesize=12.0 K 2024-11-19T21:42:21,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for abb063485168bef9628960f3efec5fb9 in 1237ms, sequenceid=289, compaction requested=true 2024-11-19T21:42:21,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:21,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:21,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:21,960 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:21,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:21,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:21,960 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:21,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:21,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:21,961 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:21,961 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:21,961 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/B is initiating minor compaction (all files) 2024-11-19T21:42:21,961 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/A is initiating minor compaction (all files) 2024-11-19T21:42:21,961 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/B in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,961 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/A in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,962 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a78fd35cf9c743969d885e888bde8290, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/13f6eaa27e19453093dd2c7c655ce542, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/5270c790c441450494984808300b6a9d] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=38.9 K 2024-11-19T21:42:21,962 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/39b3186d35084974992567866c0dbfbd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0f0f795e4c0a488da464126ea3d0dae8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/768538c4d4734424aa2019124732adaf] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=36.5 K 2024-11-19T21:42:21,962 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting a78fd35cf9c743969d885e888bde8290, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732052538524 2024-11-19T21:42:21,962 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b3186d35084974992567866c0dbfbd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732052538524 2024-11-19T21:42:21,962 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f0f795e4c0a488da464126ea3d0dae8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732052538885 2024-11-19T21:42:21,962 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13f6eaa27e19453093dd2c7c655ce542, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732052538885 2024-11-19T21:42:21,962 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 768538c4d4734424aa2019124732adaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732052540071 2024-11-19T21:42:21,962 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5270c790c441450494984808300b6a9d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732052540071 2024-11-19T21:42:21,968 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:21,969 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/0df5b0cecc9947068d6ee81df24cfda0 is 50, key is test_row_0/B:col10/1732052540071/Put/seqid=0 2024-11-19T21:42:21,977 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#A#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:21,977 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/8996e577459b4f8f8c3a7caaed5a1010 is 50, key is test_row_0/A:col10/1732052540071/Put/seqid=0 2024-11-19T21:42:21,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742357_1533 (size=13017) 2024-11-19T21:42:21,985 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/0df5b0cecc9947068d6ee81df24cfda0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0df5b0cecc9947068d6ee81df24cfda0 2024-11-19T21:42:21,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742358_1534 (size=13017) 2024-11-19T21:42:21,988 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/B of abb063485168bef9628960f3efec5fb9 into 0df5b0cecc9947068d6ee81df24cfda0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:21,989 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:21,989 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/B, priority=13, startTime=1732052541960; duration=0sec 2024-11-19T21:42:21,989 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:21,989 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:B 2024-11-19T21:42:21,989 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:21,990 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:21,990 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/C is initiating minor compaction (all files) 2024-11-19T21:42:21,990 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/C in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:21,990 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/57ef3614b2c94a43bca8700845caa891, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e8b84a7d9bdf444eb32e8aa52c2e048c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/535acc57e97d4ec7a283e17369e6c00f] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=36.5 K 2024-11-19T21:42:21,992 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ef3614b2c94a43bca8700845caa891, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732052538524 2024-11-19T21:42:21,992 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b84a7d9bdf444eb32e8aa52c2e048c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732052538885 2024-11-19T21:42:21,992 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 535acc57e97d4ec7a283e17369e6c00f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732052540071 2024-11-19T21:42:21,993 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/8996e577459b4f8f8c3a7caaed5a1010 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8996e577459b4f8f8c3a7caaed5a1010 2024-11-19T21:42:21,996 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/A of abb063485168bef9628960f3efec5fb9 into 8996e577459b4f8f8c3a7caaed5a1010(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:21,996 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:21,996 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/A, priority=13, startTime=1732052541960; duration=0sec 2024-11-19T21:42:21,996 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:21,996 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:A 2024-11-19T21:42:21,998 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:21,998 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/3e0557e7a8474e1ab4b209f327d67802 is 50, key is test_row_0/C:col10/1732052540071/Put/seqid=0 2024-11-19T21:42:22,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742359_1535 (size=13017) 2024-11-19T21:42:22,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T21:42:22,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:22,093 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T21:42:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/eafd3cabdcc142089c341b01c4d627bc is 50, key is test_row_0/A:col10/1732052540746/Put/seqid=0 2024-11-19T21:42:22,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742360_1536 (size=12301) 2024-11-19T21:42:22,277 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T21:42:22,405 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/3e0557e7a8474e1ab4b209f327d67802 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/3e0557e7a8474e1ab4b209f327d67802 2024-11-19T21:42:22,412 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/C of abb063485168bef9628960f3efec5fb9 into 3e0557e7a8474e1ab4b209f327d67802(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:22,412 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:22,412 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/C, priority=13, startTime=1732052541960; duration=0sec 2024-11-19T21:42:22,413 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:22,413 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:C 2024-11-19T21:42:22,501 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/eafd3cabdcc142089c341b01c4d627bc 2024-11-19T21:42:22,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/ee02e41c919d44309a50ae9d90377867 is 50, key is test_row_0/B:col10/1732052540746/Put/seqid=0 2024-11-19T21:42:22,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742361_1537 (size=12301) 2024-11-19T21:42:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T21:42:22,919 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/ee02e41c919d44309a50ae9d90377867 2024-11-19T21:42:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:22,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. as already flushing 2024-11-19T21:42:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/b932c1089e2746abbba9ca8ac0dfb633 is 50, key is test_row_0/C:col10/1732052540746/Put/seqid=0 2024-11-19T21:42:22,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742362_1538 (size=12301) 2024-11-19T21:42:22,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052602947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052602957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:22,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052602957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:22,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052602958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052603068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052603068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052603078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052603078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052603278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052603278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052603288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052603288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,333 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/b932c1089e2746abbba9ca8ac0dfb633 2024-11-19T21:42:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/eafd3cabdcc142089c341b01c4d627bc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/eafd3cabdcc142089c341b01c4d627bc 2024-11-19T21:42:23,339 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/eafd3cabdcc142089c341b01c4d627bc, entries=150, sequenceid=311, filesize=12.0 K 2024-11-19T21:42:23,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/ee02e41c919d44309a50ae9d90377867 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ee02e41c919d44309a50ae9d90377867 2024-11-19T21:42:23,343 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ee02e41c919d44309a50ae9d90377867, entries=150, sequenceid=311, filesize=12.0 K 2024-11-19T21:42:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/b932c1089e2746abbba9ca8ac0dfb633 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b932c1089e2746abbba9ca8ac0dfb633 2024-11-19T21:42:23,347 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b932c1089e2746abbba9ca8ac0dfb633, entries=150, sequenceid=311, filesize=12.0 K 2024-11-19T21:42:23,348 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for abb063485168bef9628960f3efec5fb9 in 1255ms, sequenceid=311, compaction requested=false 2024-11-19T21:42:23,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:23,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:23,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-19T21:42:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-19T21:42:23,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T21:42:23,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6280 sec 2024-11-19T21:42:23,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.6320 sec 2024-11-19T21:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:23,591 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:23,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T21:42:23,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/f2751847783c49a79297b01f3eff2531 is 50, key is test_row_0/A:col10/1732052543590/Put/seqid=0 2024-11-19T21:42:23,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742363_1539 (size=17181) 2024-11-19T21:42:23,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052603621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052603625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052603626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052603631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052603738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052603738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052603740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052603741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052603948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052603948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052603948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052603948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:24,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/f2751847783c49a79297b01f3eff2531 2024-11-19T21:42:24,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/1143c37303084f629abc8e9c6f7c5553 is 50, key is test_row_0/B:col10/1732052543590/Put/seqid=0 2024-11-19T21:42:24,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742364_1540 (size=12301) 2024-11-19T21:42:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052604258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052604258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052604258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052604258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:24,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/1143c37303084f629abc8e9c6f7c5553 2024-11-19T21:42:24,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/0bcfc8b9c6cc4de4b25d97af125761fc is 50, key is test_row_0/C:col10/1732052543590/Put/seqid=0 2024-11-19T21:42:24,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742365_1541 (size=12301) 2024-11-19T21:42:24,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34098 deadline: 1732052604768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34072 deadline: 1732052604778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34100 deadline: 1732052604778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34064 deadline: 1732052604778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., hostname=707262ffa86d,42821,1732052394005, seqNum=2 because the exception is null or not the one we care about 2024-11-19T21:42:24,796 INFO [Thread-2113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2113. 2024-11-19T21:42:24,797 DEBUG [Thread-2113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:24,797 DEBUG [Thread-2113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,797 DEBUG [Thread-2113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:24,799 INFO [Thread-2109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2109. 2024-11-19T21:42:24,799 DEBUG [Thread-2109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:24,799 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,799 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:24,800 INFO [Thread-2111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2111. 2024-11-19T21:42:24,800 DEBUG [Thread-2111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:24,800 DEBUG [Thread-2111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,800 DEBUG [Thread-2111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,800 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:24,805 INFO [Thread-2115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2115. 2024-11-19T21:42:24,805 DEBUG [Thread-2115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:24,805 DEBUG [Thread-2115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,805 DEBUG [Thread-2115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:24,807 INFO [Thread-2107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2107. 2024-11-19T21:42:24,808 DEBUG [Thread-2107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:24,808 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,808 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:24,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:24,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/0bcfc8b9c6cc4de4b25d97af125761fc 2024-11-19T21:42:24,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/f2751847783c49a79297b01f3eff2531 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/f2751847783c49a79297b01f3eff2531 2024-11-19T21:42:24,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/f2751847783c49a79297b01f3eff2531, entries=250, sequenceid=331, filesize=16.8 K 2024-11-19T21:42:24,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/1143c37303084f629abc8e9c6f7c5553 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/1143c37303084f629abc8e9c6f7c5553 2024-11-19T21:42:24,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/1143c37303084f629abc8e9c6f7c5553, entries=150, sequenceid=331, filesize=12.0 K 2024-11-19T21:42:24,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/0bcfc8b9c6cc4de4b25d97af125761fc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/0bcfc8b9c6cc4de4b25d97af125761fc 2024-11-19T21:42:24,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/0bcfc8b9c6cc4de4b25d97af125761fc, entries=150, sequenceid=331, filesize=12.0 K 2024-11-19T21:42:24,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for abb063485168bef9628960f3efec5fb9 in 1249ms, sequenceid=331, compaction requested=true 2024-11-19T21:42:24,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:24,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:24,840 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:24,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:24,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:24,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:24,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store abb063485168bef9628960f3efec5fb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:24,840 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:24,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:24,841 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:24,841 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:24,841 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/A is initiating minor compaction (all files) 2024-11-19T21:42:24,841 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/B is initiating minor compaction (all files) 2024-11-19T21:42:24,841 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/A in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:24,841 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/B in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:24,841 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0df5b0cecc9947068d6ee81df24cfda0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ee02e41c919d44309a50ae9d90377867, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/1143c37303084f629abc8e9c6f7c5553] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=36.7 K 2024-11-19T21:42:24,841 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8996e577459b4f8f8c3a7caaed5a1010, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/eafd3cabdcc142089c341b01c4d627bc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/f2751847783c49a79297b01f3eff2531] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=41.5 K 2024-11-19T21:42:24,842 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df5b0cecc9947068d6ee81df24cfda0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732052540071 2024-11-19T21:42:24,842 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8996e577459b4f8f8c3a7caaed5a1010, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732052540071 2024-11-19T21:42:24,842 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting eafd3cabdcc142089c341b01c4d627bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732052540746 2024-11-19T21:42:24,842 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting ee02e41c919d44309a50ae9d90377867, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732052540746 2024-11-19T21:42:24,842 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2751847783c49a79297b01f3eff2531, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732052542932 2024-11-19T21:42:24,842 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 1143c37303084f629abc8e9c6f7c5553, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732052542956 2024-11-19T21:42:24,848 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:24,848 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#A#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:24,848 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/76cf527275394e72a95aacfaa9334964 is 50, key is test_row_0/B:col10/1732052543590/Put/seqid=0 2024-11-19T21:42:24,849 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/df635bf9b34d419fb6e180d220a2d298 is 50, key is test_row_0/A:col10/1732052543590/Put/seqid=0 2024-11-19T21:42:24,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742367_1543 (size=13119) 2024-11-19T21:42:24,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742366_1542 (size=13119) 2024-11-19T21:42:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T21:42:24,878 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:25,258 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/76cf527275394e72a95aacfaa9334964 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/76cf527275394e72a95aacfaa9334964 2024-11-19T21:42:25,258 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/df635bf9b34d419fb6e180d220a2d298 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/df635bf9b34d419fb6e180d220a2d298 2024-11-19T21:42:25,265 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/A of abb063485168bef9628960f3efec5fb9 into df635bf9b34d419fb6e180d220a2d298(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:25,265 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/B of abb063485168bef9628960f3efec5fb9 into 76cf527275394e72a95aacfaa9334964(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:25,265 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:25,265 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:25,265 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/B, priority=13, startTime=1732052544840; duration=0sec 2024-11-19T21:42:25,265 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/A, priority=13, startTime=1732052544840; duration=0sec 2024-11-19T21:42:25,265 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:25,265 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:25,265 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:A 2024-11-19T21:42:25,265 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:B 2024-11-19T21:42:25,265 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:25,266 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:25,266 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): abb063485168bef9628960f3efec5fb9/C is initiating minor compaction (all files) 2024-11-19T21:42:25,266 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of abb063485168bef9628960f3efec5fb9/C in TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:25,266 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/3e0557e7a8474e1ab4b209f327d67802, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b932c1089e2746abbba9ca8ac0dfb633, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/0bcfc8b9c6cc4de4b25d97af125761fc] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp, totalSize=36.7 K 2024-11-19T21:42:25,266 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e0557e7a8474e1ab4b209f327d67802, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732052540071 2024-11-19T21:42:25,267 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting b932c1089e2746abbba9ca8ac0dfb633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732052540746 2024-11-19T21:42:25,267 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bcfc8b9c6cc4de4b25d97af125761fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732052542956 2024-11-19T21:42:25,273 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): abb063485168bef9628960f3efec5fb9#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:25,273 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/8d96642df2374bc5b0a4a133aa198b77 is 50, key is test_row_0/C:col10/1732052543590/Put/seqid=0 2024-11-19T21:42:25,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742368_1544 (size=13119) 2024-11-19T21:42:25,686 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/8d96642df2374bc5b0a4a133aa198b77 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/8d96642df2374bc5b0a4a133aa198b77 2024-11-19T21:42:25,691 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in abb063485168bef9628960f3efec5fb9/C of abb063485168bef9628960f3efec5fb9 into 8d96642df2374bc5b0a4a133aa198b77(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:25,691 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:25,691 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., storeName=abb063485168bef9628960f3efec5fb9/C, priority=13, startTime=1732052544840; duration=0sec 2024-11-19T21:42:25,691 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:25,691 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: abb063485168bef9628960f3efec5fb9:C 2024-11-19T21:42:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:25,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:25,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T21:42:25,790 INFO [Thread-2102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2102. 2024-11-19T21:42:25,790 DEBUG [Thread-2102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:25,790 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:25,790 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:25,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:25,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/993009529c9045fb94de2e9705d97aef is 50, key is test_row_0/A:col10/1732052545788/Put/seqid=0 2024-11-19T21:42:25,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742369_1545 (size=12301) 2024-11-19T21:42:25,801 INFO [Thread-2098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2098. 2024-11-19T21:42:25,801 INFO [Thread-2104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2104. 2024-11-19T21:42:25,801 INFO [Thread-2100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2100. 2024-11-19T21:42:25,801 DEBUG [Thread-2098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:25,801 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:25,801 DEBUG [Thread-2100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:25,801 DEBUG [Thread-2104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:25,801 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:25,801 DEBUG [Thread-2104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:25,801 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:25,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:25,801 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:25,801 DEBUG [Thread-2104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:25,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:25,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:26,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/993009529c9045fb94de2e9705d97aef 2024-11-19T21:42:26,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/f99d858f9f3d434f9f5a0c27b4c620ef is 50, key is test_row_0/B:col10/1732052545788/Put/seqid=0 2024-11-19T21:42:26,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742370_1546 (size=12301) 2024-11-19T21:42:26,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/f99d858f9f3d434f9f5a0c27b4c620ef 2024-11-19T21:42:26,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/db011b0556ae4c1c8c7680b1d9da0a94 is 50, key is test_row_0/C:col10/1732052545788/Put/seqid=0 2024-11-19T21:42:26,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742371_1547 (size=12301) 2024-11-19T21:42:27,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/db011b0556ae4c1c8c7680b1d9da0a94 2024-11-19T21:42:27,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/993009529c9045fb94de2e9705d97aef as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/993009529c9045fb94de2e9705d97aef 2024-11-19T21:42:27,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/993009529c9045fb94de2e9705d97aef, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T21:42:27,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/f99d858f9f3d434f9f5a0c27b4c620ef as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/f99d858f9f3d434f9f5a0c27b4c620ef 2024-11-19T21:42:27,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/f99d858f9f3d434f9f5a0c27b4c620ef, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T21:42:27,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/db011b0556ae4c1c8c7680b1d9da0a94 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/db011b0556ae4c1c8c7680b1d9da0a94 2024-11-19T21:42:27,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/db011b0556ae4c1c8c7680b1d9da0a94, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T21:42:27,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=20.13 KB/20610 for abb063485168bef9628960f3efec5fb9 in 1255ms, sequenceid=352, compaction requested=false 2024-11-19T21:42:27,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for abb063485168bef9628960f3efec5fb9: 2024-11-19T21:42:29,773 INFO [Thread-2096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2096. 2024-11-19T21:42:29,773 DEBUG [Thread-2096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:29,773 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:29,773 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:29,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:29,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T21:42:29,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-19T21:42:29,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-19T21:42:29,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-19T21:42:29,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-19T21:42:29,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-19T21:42:29,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T21:42:29,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1550 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4644 rows 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1554 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4660 rows 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1551 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4653 rows 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1547 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4638 rows 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1549 2024-11-19T21:42:29,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4647 rows 2024-11-19T21:42:29,775 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:42:29,776 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:42:29,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:29,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:29,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:29,778 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T21:42:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T21:42:29,785 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052549784"}]},"ts":"1732052549784"} 2024-11-19T21:42:29,786 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T21:42:29,786 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T21:42:29,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T21:42:29,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abb063485168bef9628960f3efec5fb9, UNASSIGN}] 2024-11-19T21:42:29,790 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abb063485168bef9628960f3efec5fb9, UNASSIGN 2024-11-19T21:42:29,791 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=abb063485168bef9628960f3efec5fb9, regionState=CLOSING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:42:29,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=abb063485168bef9628960f3efec5fb9, UNASSIGN because future has completed 2024-11-19T21:42:29,793 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T21:42:29,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:42:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T21:42:29,947 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:29,947 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T21:42:29,947 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing abb063485168bef9628960f3efec5fb9, disabling compactions & flushes 2024-11-19T21:42:29,948 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:29,948 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:29,948 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. after waiting 0 ms 2024-11-19T21:42:29,948 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:29,948 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing abb063485168bef9628960f3efec5fb9 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-19T21:42:29,958 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/532aabc3f3ea4d22b31f5a383b94a670 is 50, key is test_row_0/A:col10/1732052545799/Put/seqid=0 2024-11-19T21:42:29,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742372_1548 (size=12301) 2024-11-19T21:42:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T21:42:30,364 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/532aabc3f3ea4d22b31f5a383b94a670 2024-11-19T21:42:30,373 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/8b462848ec0a4db08b18b4ae6fde7fd2 is 50, key is test_row_0/B:col10/1732052545799/Put/seqid=0 2024-11-19T21:42:30,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742373_1549 (size=12301) 2024-11-19T21:42:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T21:42:30,779 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/8b462848ec0a4db08b18b4ae6fde7fd2 2024-11-19T21:42:30,793 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/9373519521a54e4aa9aaeb0666195d3f is 50, key is test_row_0/C:col10/1732052545799/Put/seqid=0 2024-11-19T21:42:30,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742374_1550 (size=12301) 2024-11-19T21:42:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T21:42:31,199 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/9373519521a54e4aa9aaeb0666195d3f 2024-11-19T21:42:31,208 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/A/532aabc3f3ea4d22b31f5a383b94a670 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/532aabc3f3ea4d22b31f5a383b94a670 2024-11-19T21:42:31,212 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/532aabc3f3ea4d22b31f5a383b94a670, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T21:42:31,213 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/B/8b462848ec0a4db08b18b4ae6fde7fd2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/8b462848ec0a4db08b18b4ae6fde7fd2 2024-11-19T21:42:31,216 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/8b462848ec0a4db08b18b4ae6fde7fd2, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T21:42:31,216 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/.tmp/C/9373519521a54e4aa9aaeb0666195d3f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/9373519521a54e4aa9aaeb0666195d3f 2024-11-19T21:42:31,220 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/9373519521a54e4aa9aaeb0666195d3f, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T21:42:31,221 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for abb063485168bef9628960f3efec5fb9 in 1273ms, sequenceid=359, compaction requested=true 2024-11-19T21:42:31,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b8aef36950ce44c8b069d8ea8d5be576, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4b255e7814624b51a5e1addf7230e5fa, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/7e51ab78cd0e43498216e300940b8c4c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/3a15f86c449947108422ebcdcd700c62, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/23d4bb84103a4f5fbfccb76b8c927b42, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/28f994186b9b4a8290363d99ee4eda2d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/6e2ca4ab25e6469194a28dc426cf0945, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/75c675cffa704bcb8493cf1a09be45b0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8325ccb58e5d4b33b1c18fc45763ced4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4245f5b8ac204fdba9726c8ed3110fb1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cbe1d90fd7c7464b9cda7dd48d795eb6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4fdc8fb942ea4a8eae9b31e220561c30, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/9e7e54366e4c4a8a9ee9024d123f36bd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a838a4c10fbb488b8f4a0c4b10f10a3d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/866644b8752a470d97b6e296ef0dde22, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cacf5adb70484043adf5b091fa2a1487, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/167bfe0f6f7740bca1727784a04c3f52, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b39af114c0d6456a82718ab0c9109817, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a78fd35cf9c743969d885e888bde8290, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/13f6eaa27e19453093dd2c7c655ce542, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/5270c790c441450494984808300b6a9d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8996e577459b4f8f8c3a7caaed5a1010, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/eafd3cabdcc142089c341b01c4d627bc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/f2751847783c49a79297b01f3eff2531] to archive 2024-11-19T21:42:31,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:42:31,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b8aef36950ce44c8b069d8ea8d5be576 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b8aef36950ce44c8b069d8ea8d5be576 2024-11-19T21:42:31,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4b255e7814624b51a5e1addf7230e5fa to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4b255e7814624b51a5e1addf7230e5fa 2024-11-19T21:42:31,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/7e51ab78cd0e43498216e300940b8c4c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/7e51ab78cd0e43498216e300940b8c4c 2024-11-19T21:42:31,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/3a15f86c449947108422ebcdcd700c62 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/3a15f86c449947108422ebcdcd700c62 2024-11-19T21:42:31,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/23d4bb84103a4f5fbfccb76b8c927b42 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/23d4bb84103a4f5fbfccb76b8c927b42 2024-11-19T21:42:31,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/28f994186b9b4a8290363d99ee4eda2d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/28f994186b9b4a8290363d99ee4eda2d 2024-11-19T21:42:31,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/6e2ca4ab25e6469194a28dc426cf0945 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/6e2ca4ab25e6469194a28dc426cf0945 2024-11-19T21:42:31,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/75c675cffa704bcb8493cf1a09be45b0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/75c675cffa704bcb8493cf1a09be45b0 2024-11-19T21:42:31,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8325ccb58e5d4b33b1c18fc45763ced4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8325ccb58e5d4b33b1c18fc45763ced4 2024-11-19T21:42:31,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4245f5b8ac204fdba9726c8ed3110fb1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4245f5b8ac204fdba9726c8ed3110fb1 2024-11-19T21:42:31,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cbe1d90fd7c7464b9cda7dd48d795eb6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cbe1d90fd7c7464b9cda7dd48d795eb6 2024-11-19T21:42:31,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4fdc8fb942ea4a8eae9b31e220561c30 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/4fdc8fb942ea4a8eae9b31e220561c30 2024-11-19T21:42:31,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/9e7e54366e4c4a8a9ee9024d123f36bd to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/9e7e54366e4c4a8a9ee9024d123f36bd 2024-11-19T21:42:31,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a838a4c10fbb488b8f4a0c4b10f10a3d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a838a4c10fbb488b8f4a0c4b10f10a3d 2024-11-19T21:42:31,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/866644b8752a470d97b6e296ef0dde22 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/866644b8752a470d97b6e296ef0dde22 2024-11-19T21:42:31,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cacf5adb70484043adf5b091fa2a1487 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/cacf5adb70484043adf5b091fa2a1487 2024-11-19T21:42:31,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/167bfe0f6f7740bca1727784a04c3f52 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/167bfe0f6f7740bca1727784a04c3f52 2024-11-19T21:42:31,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b39af114c0d6456a82718ab0c9109817 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/b39af114c0d6456a82718ab0c9109817 2024-11-19T21:42:31,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a78fd35cf9c743969d885e888bde8290 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/a78fd35cf9c743969d885e888bde8290 2024-11-19T21:42:31,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/13f6eaa27e19453093dd2c7c655ce542 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/13f6eaa27e19453093dd2c7c655ce542 2024-11-19T21:42:31,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/5270c790c441450494984808300b6a9d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/5270c790c441450494984808300b6a9d 2024-11-19T21:42:31,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8996e577459b4f8f8c3a7caaed5a1010 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/8996e577459b4f8f8c3a7caaed5a1010 2024-11-19T21:42:31,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/eafd3cabdcc142089c341b01c4d627bc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/eafd3cabdcc142089c341b01c4d627bc 2024-11-19T21:42:31,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/f2751847783c49a79297b01f3eff2531 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/f2751847783c49a79297b01f3eff2531 2024-11-19T21:42:31,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/583590b783cb47a0928d78f34a57e29d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e585a4ef35ed4352b926c9dcc2dde392, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/4dbfca20c1b041218b92f449ee5a9010, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/c18e117beaba460db9080d1d42abcb0c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e064f15e7af54ed7bfb2cbae2875389e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ba2bfad7678b416badabea9cc31152ad, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0326f7b0ff104a3c8b6f3dd9bfe5e376, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/415e592728fd455e9faac965e85710fb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6def325aed024d7fbab4a8d9bf965e8b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/df9cad67b00345ac882d7fd68eef7ab4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/11597c218fc3476cb2e772828011308c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6269160f2daf4291aa00e1d0eae68d7e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/b9960b50dd404b1098540193a1a6c4b0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/885d7ee7294c4309b9fe3cdd36104278, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/2c42262bd77841b2a30f063348fbf8ce, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/bfb2c7b2aac042e68cbba4c40ac6ea0c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/5831119008ef4870913dd6ce2186cc42, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/39b3186d35084974992567866c0dbfbd, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/61f271cc772b441d98fde7c50db7688d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0f0f795e4c0a488da464126ea3d0dae8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0df5b0cecc9947068d6ee81df24cfda0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/768538c4d4734424aa2019124732adaf, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ee02e41c919d44309a50ae9d90377867, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/1143c37303084f629abc8e9c6f7c5553] to archive 2024-11-19T21:42:31,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:42:31,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/583590b783cb47a0928d78f34a57e29d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/583590b783cb47a0928d78f34a57e29d 2024-11-19T21:42:31,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e585a4ef35ed4352b926c9dcc2dde392 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e585a4ef35ed4352b926c9dcc2dde392 2024-11-19T21:42:31,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/4dbfca20c1b041218b92f449ee5a9010 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/4dbfca20c1b041218b92f449ee5a9010 2024-11-19T21:42:31,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/c18e117beaba460db9080d1d42abcb0c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/c18e117beaba460db9080d1d42abcb0c 2024-11-19T21:42:31,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e064f15e7af54ed7bfb2cbae2875389e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/e064f15e7af54ed7bfb2cbae2875389e 2024-11-19T21:42:31,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ba2bfad7678b416badabea9cc31152ad to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ba2bfad7678b416badabea9cc31152ad 2024-11-19T21:42:31,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0326f7b0ff104a3c8b6f3dd9bfe5e376 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0326f7b0ff104a3c8b6f3dd9bfe5e376 2024-11-19T21:42:31,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/415e592728fd455e9faac965e85710fb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/415e592728fd455e9faac965e85710fb 2024-11-19T21:42:31,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6def325aed024d7fbab4a8d9bf965e8b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6def325aed024d7fbab4a8d9bf965e8b 2024-11-19T21:42:31,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/df9cad67b00345ac882d7fd68eef7ab4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/df9cad67b00345ac882d7fd68eef7ab4 2024-11-19T21:42:31,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/11597c218fc3476cb2e772828011308c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/11597c218fc3476cb2e772828011308c 2024-11-19T21:42:31,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6269160f2daf4291aa00e1d0eae68d7e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/6269160f2daf4291aa00e1d0eae68d7e 2024-11-19T21:42:31,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/b9960b50dd404b1098540193a1a6c4b0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/b9960b50dd404b1098540193a1a6c4b0 2024-11-19T21:42:31,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/885d7ee7294c4309b9fe3cdd36104278 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/885d7ee7294c4309b9fe3cdd36104278 2024-11-19T21:42:31,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/2c42262bd77841b2a30f063348fbf8ce to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/2c42262bd77841b2a30f063348fbf8ce 2024-11-19T21:42:31,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/bfb2c7b2aac042e68cbba4c40ac6ea0c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/bfb2c7b2aac042e68cbba4c40ac6ea0c 2024-11-19T21:42:31,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/5831119008ef4870913dd6ce2186cc42 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/5831119008ef4870913dd6ce2186cc42 2024-11-19T21:42:31,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/39b3186d35084974992567866c0dbfbd to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/39b3186d35084974992567866c0dbfbd 2024-11-19T21:42:31,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/61f271cc772b441d98fde7c50db7688d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/61f271cc772b441d98fde7c50db7688d 2024-11-19T21:42:31,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0f0f795e4c0a488da464126ea3d0dae8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0f0f795e4c0a488da464126ea3d0dae8 2024-11-19T21:42:31,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0df5b0cecc9947068d6ee81df24cfda0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/0df5b0cecc9947068d6ee81df24cfda0 2024-11-19T21:42:31,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/768538c4d4734424aa2019124732adaf to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/768538c4d4734424aa2019124732adaf 2024-11-19T21:42:31,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ee02e41c919d44309a50ae9d90377867 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/ee02e41c919d44309a50ae9d90377867 2024-11-19T21:42:31,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/1143c37303084f629abc8e9c6f7c5553 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/1143c37303084f629abc8e9c6f7c5553 2024-11-19T21:42:31,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b21016646e3043dd960d469fb8d80414, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/14d7a9519e9d464f91f57a81ea80cce0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/c96dbeb9fe29474aa7651dda7e3b1bd6, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/a4621f1dd323442780e0ea96718c3fab, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/df3da3c56f68476d8349f993cf104679, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/be06636f73e94cb8bea790e118f78265, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/d43cead540c240ccaaeeb7b6bb8a3791, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/675191db32cf4364845220ec801edc3c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1882c4f12453478e92005206d2eec5d4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b2c94f96676b4c99a6a32092d5b96ffb, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e29398bf9cc3412fa2474619b2dd6e42, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/2a90f28d9ba446559a1e0452c403ab03, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/ff1fe2bf08474daf93baae62c0c11df5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/43a929b70db6414b8bb8c31b46fd9bed, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/8f1e9610a71549789596892c9f3d415b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1e48b34cf6394ab79057d8752b2d22b7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/f8e3cbb471cb4494ba68b5b2dd59f44d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/57ef3614b2c94a43bca8700845caa891, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/001a8ac3ae4546a9b43c08e6ae372e92, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e8b84a7d9bdf444eb32e8aa52c2e048c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/3e0557e7a8474e1ab4b209f327d67802, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/535acc57e97d4ec7a283e17369e6c00f, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b932c1089e2746abbba9ca8ac0dfb633, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/0bcfc8b9c6cc4de4b25d97af125761fc] to archive 2024-11-19T21:42:31,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:42:31,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b21016646e3043dd960d469fb8d80414 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b21016646e3043dd960d469fb8d80414 2024-11-19T21:42:31,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/14d7a9519e9d464f91f57a81ea80cce0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/14d7a9519e9d464f91f57a81ea80cce0 2024-11-19T21:42:31,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/c96dbeb9fe29474aa7651dda7e3b1bd6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/c96dbeb9fe29474aa7651dda7e3b1bd6 2024-11-19T21:42:31,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/a4621f1dd323442780e0ea96718c3fab to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/a4621f1dd323442780e0ea96718c3fab 2024-11-19T21:42:31,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/df3da3c56f68476d8349f993cf104679 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/df3da3c56f68476d8349f993cf104679 2024-11-19T21:42:31,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/be06636f73e94cb8bea790e118f78265 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/be06636f73e94cb8bea790e118f78265 2024-11-19T21:42:31,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/d43cead540c240ccaaeeb7b6bb8a3791 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/d43cead540c240ccaaeeb7b6bb8a3791 2024-11-19T21:42:31,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/675191db32cf4364845220ec801edc3c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/675191db32cf4364845220ec801edc3c 2024-11-19T21:42:31,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1882c4f12453478e92005206d2eec5d4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1882c4f12453478e92005206d2eec5d4 2024-11-19T21:42:31,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b2c94f96676b4c99a6a32092d5b96ffb to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b2c94f96676b4c99a6a32092d5b96ffb 2024-11-19T21:42:31,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e29398bf9cc3412fa2474619b2dd6e42 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e29398bf9cc3412fa2474619b2dd6e42 2024-11-19T21:42:31,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/2a90f28d9ba446559a1e0452c403ab03 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/2a90f28d9ba446559a1e0452c403ab03 2024-11-19T21:42:31,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/ff1fe2bf08474daf93baae62c0c11df5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/ff1fe2bf08474daf93baae62c0c11df5 2024-11-19T21:42:31,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/43a929b70db6414b8bb8c31b46fd9bed to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/43a929b70db6414b8bb8c31b46fd9bed 2024-11-19T21:42:31,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/8f1e9610a71549789596892c9f3d415b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/8f1e9610a71549789596892c9f3d415b 2024-11-19T21:42:31,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1e48b34cf6394ab79057d8752b2d22b7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/1e48b34cf6394ab79057d8752b2d22b7 2024-11-19T21:42:31,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/f8e3cbb471cb4494ba68b5b2dd59f44d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/f8e3cbb471cb4494ba68b5b2dd59f44d 2024-11-19T21:42:31,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/57ef3614b2c94a43bca8700845caa891 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/57ef3614b2c94a43bca8700845caa891 2024-11-19T21:42:31,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/001a8ac3ae4546a9b43c08e6ae372e92 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/001a8ac3ae4546a9b43c08e6ae372e92 2024-11-19T21:42:31,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e8b84a7d9bdf444eb32e8aa52c2e048c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/e8b84a7d9bdf444eb32e8aa52c2e048c 2024-11-19T21:42:31,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/3e0557e7a8474e1ab4b209f327d67802 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/3e0557e7a8474e1ab4b209f327d67802 2024-11-19T21:42:31,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/535acc57e97d4ec7a283e17369e6c00f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/535acc57e97d4ec7a283e17369e6c00f 2024-11-19T21:42:31,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b932c1089e2746abbba9ca8ac0dfb633 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/b932c1089e2746abbba9ca8ac0dfb633 2024-11-19T21:42:31,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/0bcfc8b9c6cc4de4b25d97af125761fc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/0bcfc8b9c6cc4de4b25d97af125761fc 2024-11-19T21:42:31,298 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/recovered.edits/362.seqid, newMaxSeqId=362, maxSeqId=1 2024-11-19T21:42:31,298 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. 2024-11-19T21:42:31,299 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for abb063485168bef9628960f3efec5fb9: Waiting for close lock at 1732052549947Running coprocessor pre-close hooks at 1732052549947Disabling compacts and flushes for region at 1732052549947Disabling writes for close at 1732052549948 (+1 ms)Obtaining lock to block concurrent updates at 1732052549948Preparing flush snapshotting stores in abb063485168bef9628960f3efec5fb9 at 1732052549948Finished memstore snapshotting TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732052549949 (+1 ms)Flushing stores of TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9. at 1732052549950 (+1 ms)Flushing abb063485168bef9628960f3efec5fb9/A: creating writer at 1732052549950Flushing abb063485168bef9628960f3efec5fb9/A: appending metadata at 1732052549957 (+7 ms)Flushing abb063485168bef9628960f3efec5fb9/A: closing flushed file at 1732052549957Flushing abb063485168bef9628960f3efec5fb9/B: creating writer at 1732052550370 (+413 ms)Flushing abb063485168bef9628960f3efec5fb9/B: appending metadata at 1732052550373 (+3 ms)Flushing abb063485168bef9628960f3efec5fb9/B: closing flushed file at 1732052550373Flushing abb063485168bef9628960f3efec5fb9/C: creating writer at 1732052550790 (+417 ms)Flushing abb063485168bef9628960f3efec5fb9/C: appending metadata at 1732052550793 (+3 ms)Flushing abb063485168bef9628960f3efec5fb9/C: closing flushed file at 1732052550793Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20677653: reopening flushed file at 1732052551208 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e81e3d6: reopening flushed file at 1732052551212 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b55f2: reopening flushed file at 1732052551216 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for abb063485168bef9628960f3efec5fb9 in 1273ms, sequenceid=359, compaction requested=true at 1732052551221 (+5 ms)Writing region close event to WAL at 1732052551296 (+75 ms)Running coprocessor post-close hooks at 1732052551298 (+2 ms)Closed at 1732052551298 2024-11-19T21:42:31,300 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:31,301 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=abb063485168bef9628960f3efec5fb9, regionState=CLOSED 2024-11-19T21:42:31,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:42:31,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T21:42:31,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure abb063485168bef9628960f3efec5fb9, server=707262ffa86d,42821,1732052394005 in 1.5100 sec 2024-11-19T21:42:31,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-19T21:42:31,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=abb063485168bef9628960f3efec5fb9, UNASSIGN in 1.5150 sec 2024-11-19T21:42:31,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T21:42:31,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5190 sec 2024-11-19T21:42:31,308 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052551308"}]},"ts":"1732052551308"} 2024-11-19T21:42:31,309 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T21:42:31,309 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T21:42:31,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-19T21:42:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T21:42:31,929 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:31,931 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T21:42:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:31,936 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T21:42:31,937 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:31,940 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:31,944 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/recovered.edits] 2024-11-19T21:42:31,948 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/532aabc3f3ea4d22b31f5a383b94a670 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/532aabc3f3ea4d22b31f5a383b94a670 2024-11-19T21:42:31,951 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/993009529c9045fb94de2e9705d97aef to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/993009529c9045fb94de2e9705d97aef 2024-11-19T21:42:31,953 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/df635bf9b34d419fb6e180d220a2d298 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/A/df635bf9b34d419fb6e180d220a2d298 2024-11-19T21:42:31,957 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/76cf527275394e72a95aacfaa9334964 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/76cf527275394e72a95aacfaa9334964 2024-11-19T21:42:31,958 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/8b462848ec0a4db08b18b4ae6fde7fd2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/8b462848ec0a4db08b18b4ae6fde7fd2 2024-11-19T21:42:31,959 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/f99d858f9f3d434f9f5a0c27b4c620ef to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/B/f99d858f9f3d434f9f5a0c27b4c620ef 2024-11-19T21:42:31,961 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/8d96642df2374bc5b0a4a133aa198b77 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/8d96642df2374bc5b0a4a133aa198b77 2024-11-19T21:42:31,962 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/9373519521a54e4aa9aaeb0666195d3f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/9373519521a54e4aa9aaeb0666195d3f 2024-11-19T21:42:31,963 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/db011b0556ae4c1c8c7680b1d9da0a94 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/C/db011b0556ae4c1c8c7680b1d9da0a94 2024-11-19T21:42:31,965 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/recovered.edits/362.seqid to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9/recovered.edits/362.seqid 2024-11-19T21:42:31,965 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/abb063485168bef9628960f3efec5fb9 2024-11-19T21:42:31,965 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T21:42:31,967 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:31,968 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T21:42:31,970 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T21:42:31,971 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:31,971 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T21:42:31,971 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732052551971"}]},"ts":"9223372036854775807"} 2024-11-19T21:42:31,972 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T21:42:31,972 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => abb063485168bef9628960f3efec5fb9, NAME => 'TestAcidGuarantees,,1732052522599.abb063485168bef9628960f3efec5fb9.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T21:42:31,972 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T21:42:31,972 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732052551972"}]},"ts":"9223372036854775807"} 2024-11-19T21:42:31,974 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T21:42:31,974 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:31,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-19T21:42:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T21:42:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T21:42:32,049 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:32,059 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239 (was 242), OpenFileDescriptor=447 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 341) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4799 (was 5031) 2024-11-19T21:42:32,069 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=368, ProcessCount=11, AvailableMemoryMB=4799 2024-11-19T21:42:32,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T21:42:32,070 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:42:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:32,072 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T21:42:32,073 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:32,073 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-19T21:42:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T21:42:32,073 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T21:42:32,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742375_1551 (size=959) 2024-11-19T21:42:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T21:42:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T21:42:32,483 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b 2024-11-19T21:42:32,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742376_1552 (size=53) 2024-11-19T21:42:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T21:42:32,893 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:42:32,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8b95a52d5be5c4d97b13c59880275ce5, disabling compactions & flushes 2024-11-19T21:42:32,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:32,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:32,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. after waiting 0 ms 2024-11-19T21:42:32,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:32,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:32,894 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b95a52d5be5c4d97b13c59880275ce5: Waiting for close lock at 1732052552893Disabling compacts and flushes for region at 1732052552893Disabling writes for close at 1732052552894 (+1 ms)Writing region close event to WAL at 1732052552894Closed at 1732052552894 2024-11-19T21:42:32,895 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T21:42:32,896 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732052552896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732052552896"}]},"ts":"1732052552896"} 2024-11-19T21:42:32,898 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T21:42:32,900 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T21:42:32,900 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052552900"}]},"ts":"1732052552900"} 2024-11-19T21:42:32,902 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T21:42:32,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, ASSIGN}] 2024-11-19T21:42:32,903 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, ASSIGN 2024-11-19T21:42:32,904 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, ASSIGN; state=OFFLINE, location=707262ffa86d,42821,1732052394005; forceNewPlan=false, retain=false 2024-11-19T21:42:33,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=8b95a52d5be5c4d97b13c59880275ce5, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:42:33,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, ASSIGN because future has completed 2024-11-19T21:42:33,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:42:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T21:42:33,219 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:33,219 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:42:33,220 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,220 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:42:33,220 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,221 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,222 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,224 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b95a52d5be5c4d97b13c59880275ce5 columnFamilyName A 2024-11-19T21:42:33,224 DEBUG [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:33,225 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(327): Store=8b95a52d5be5c4d97b13c59880275ce5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:42:33,225 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,226 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b95a52d5be5c4d97b13c59880275ce5 columnFamilyName B 2024-11-19T21:42:33,226 DEBUG [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:33,227 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(327): Store=8b95a52d5be5c4d97b13c59880275ce5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:42:33,227 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,229 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b95a52d5be5c4d97b13c59880275ce5 columnFamilyName C 2024-11-19T21:42:33,229 DEBUG [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:33,229 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(327): Store=8b95a52d5be5c4d97b13c59880275ce5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:42:33,230 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,231 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,231 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,232 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,233 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,233 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:42:33,234 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,236 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T21:42:33,236 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 8b95a52d5be5c4d97b13c59880275ce5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69677667, jitterRate=0.03827814757823944}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:42:33,236 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:33,236 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 8b95a52d5be5c4d97b13c59880275ce5: Running coprocessor pre-open hook at 1732052553221Writing region info on filesystem at 1732052553221Initializing all the Stores at 1732052553222 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052553222Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052553222Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052553222Cleaning up temporary data from old regions at 1732052553233 (+11 ms)Running coprocessor post-open hooks at 1732052553236 (+3 ms)Region opened successfully at 1732052553236 2024-11-19T21:42:33,237 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., pid=140, masterSystemTime=1732052553211 2024-11-19T21:42:33,238 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:33,238 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:33,239 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=8b95a52d5be5c4d97b13c59880275ce5, regionState=OPEN, openSeqNum=2, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:42:33,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:42:33,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T21:42:33,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 in 183 msec 2024-11-19T21:42:33,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-19T21:42:33,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, ASSIGN in 340 msec 2024-11-19T21:42:33,244 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T21:42:33,244 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052553244"}]},"ts":"1732052553244"} 2024-11-19T21:42:33,245 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T21:42:33,245 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T21:42:33,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-19T21:42:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-19T21:42:34,218 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:34,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4826bbfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:34,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:34,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:34,222 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@961586e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:34,225 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@118fcc69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:34,227 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:34,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:34,230 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T21:42:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 707262ffa86d,37541,1732052393169 2024-11-19T21:42:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@580dd7d9 2024-11-19T21:42:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T21:42:34,233 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T21:42:34,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T21:42:34,235 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T21:42:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T21:42:34,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742377_1553 (size=995) 2024-11-19T21:42:34,647 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T21:42:34,647 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T21:42:34,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T21:42:34,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, REOPEN/MOVE}] 2024-11-19T21:42:34,653 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, REOPEN/MOVE 2024-11-19T21:42:34,654 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8b95a52d5be5c4d97b13c59880275ce5, regionState=CLOSING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:42:34,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, REOPEN/MOVE because future has completed 2024-11-19T21:42:34,656 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T21:42:34,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:42:34,808 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:34,808 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T21:42:34,808 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 8b95a52d5be5c4d97b13c59880275ce5, disabling compactions & flushes 2024-11-19T21:42:34,808 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:34,808 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:34,808 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. after waiting 0 ms 2024-11-19T21:42:34,808 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:34,813 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T21:42:34,813 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:34,813 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 8b95a52d5be5c4d97b13c59880275ce5: Waiting for close lock at 1732052554808Running coprocessor pre-close hooks at 1732052554808Disabling compacts and flushes for region at 1732052554808Disabling writes for close at 1732052554808Writing region close event to WAL at 1732052554809 (+1 ms)Running coprocessor post-close hooks at 1732052554813 (+4 ms)Closed at 1732052554813 2024-11-19T21:42:34,813 WARN [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 8b95a52d5be5c4d97b13c59880275ce5 to self. 2024-11-19T21:42:34,815 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:34,816 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8b95a52d5be5c4d97b13c59880275ce5, regionState=CLOSED 2024-11-19T21:42:34,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:42:34,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-19T21:42:34,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 in 162 msec 2024-11-19T21:42:34,821 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, REOPEN/MOVE; state=CLOSED, location=707262ffa86d,42821,1732052394005; forceNewPlan=false, retain=true 2024-11-19T21:42:34,972 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8b95a52d5be5c4d97b13c59880275ce5, regionState=OPENING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:42:34,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, REOPEN/MOVE because future has completed 2024-11-19T21:42:34,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:42:35,135 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,135 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} 2024-11-19T21:42:35,136 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,136 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T21:42:35,137 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,137 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,139 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,140 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b95a52d5be5c4d97b13c59880275ce5 columnFamilyName A 2024-11-19T21:42:35,141 DEBUG [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:35,141 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(327): Store=8b95a52d5be5c4d97b13c59880275ce5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:42:35,142 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,142 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b95a52d5be5c4d97b13c59880275ce5 columnFamilyName B 2024-11-19T21:42:35,142 DEBUG [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:35,143 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(327): Store=8b95a52d5be5c4d97b13c59880275ce5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:42:35,143 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,144 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b95a52d5be5c4d97b13c59880275ce5 columnFamilyName C 2024-11-19T21:42:35,144 DEBUG [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:35,144 INFO [StoreOpener-8b95a52d5be5c4d97b13c59880275ce5-1 {}] regionserver.HStore(327): Store=8b95a52d5be5c4d97b13c59880275ce5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T21:42:35,144 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,145 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,146 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,147 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,147 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,147 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T21:42:35,149 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,150 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 8b95a52d5be5c4d97b13c59880275ce5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67711698, jitterRate=0.008982926607131958}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T21:42:35,150 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,150 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 8b95a52d5be5c4d97b13c59880275ce5: Running coprocessor pre-open hook at 1732052555137Writing region info on filesystem at 1732052555137Initializing all the Stores at 1732052555138 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052555138Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052555139 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732052555139Cleaning up temporary data from old regions at 1732052555147 (+8 ms)Running coprocessor post-open hooks at 1732052555150 (+3 ms)Region opened successfully at 1732052555150 2024-11-19T21:42:35,151 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., pid=145, masterSystemTime=1732052555128 2024-11-19T21:42:35,152 DEBUG [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,152 INFO [RS_OPEN_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,152 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8b95a52d5be5c4d97b13c59880275ce5, regionState=OPEN, openSeqNum=5, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:42:35,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-19T21:42:35,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 in 179 msec 2024-11-19T21:42:35,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-19T21:42:35,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, REOPEN/MOVE in 504 msec 2024-11-19T21:42:35,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T21:42:35,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-19T21:42:35,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 925 msec 2024-11-19T21:42:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T21:42:35,162 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:35,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604e524e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,164 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdd29b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,165 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5decb58c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f36d46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,167 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a7a401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,168 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f54b9e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5e61df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,170 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@174bf290, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cab45e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4885acdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,173 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ac7820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc1b2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc99027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,176 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4690f7c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,177 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@124b04f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3420b4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,178 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5834a95b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,179 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4441bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1b2f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,181 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229b59c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,182 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@592eeb8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1102b71d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,184 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c0d690b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,185 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3e128d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fb506a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,187 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747a987f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,188 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75dd4e94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179b5e7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 707262ffa86d,37541,-1 for getting cluster id 2024-11-19T21:42:35,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T21:42:35,189 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18a7bc1d-40c6-490c-9d94-197e3bb518aa' 2024-11-19T21:42:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T21:42:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18a7bc1d-40c6-490c-9d94-197e3bb518aa" 2024-11-19T21:42:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd21019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [707262ffa86d,37541,-1] 2024-11-19T21:42:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T21:42:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:35,190 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T21:42:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@184bf2eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T21:42:35,191 DEBUG [Thread-2468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,191 DEBUG [Thread-2477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,191 DEBUG [Thread-2462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,191 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,192 DEBUG [Thread-2460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,192 DEBUG [Thread-2479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,192 DEBUG [Thread-2466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,192 DEBUG [Thread-2464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,193 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,193 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,194 DEBUG [Thread-2464 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,194 DEBUG [Thread-2466 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-19T21:42:35,195 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T21:42:35,195 DEBUG [Thread-2464 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T21:42:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=707262ffa86d,42821,1732052394005, seqNum=-1] 2024-11-19T21:42:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T21:42:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T21:42:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,199 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5] 2024-11-19T21:42:35,200 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:35,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,204 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:35,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:42:35,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119087227ac03724af5a0ceb1531d1f9851_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052555202/Put/seqid=0 2024-11-19T21:42:35,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052615226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052615227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742378_1554 (size=14594) 2024-11-19T21:42:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052615229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052615229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052615230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T21:42:35,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052615338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052615338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052615338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052615338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052615339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:35,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:35,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T21:42:35,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052615548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052615548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052615548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052615548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052615549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,631 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:35,634 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119087227ac03724af5a0ceb1531d1f9851_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119087227ac03724af5a0ceb1531d1f9851_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:35,634 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/342ac102b61a4986bd84de2c1f237adc, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:35,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/342ac102b61a4986bd84de2c1f237adc is 175, key is test_row_0/A:col10/1732052555202/Put/seqid=0 2024-11-19T21:42:35,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742379_1555 (size=39549) 2024-11-19T21:42:35,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:35,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:35,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:35,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:35,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T21:42:35,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052615858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052615858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052615858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052615858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052615859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:35,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:35,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,039 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/342ac102b61a4986bd84de2c1f237adc 2024-11-19T21:42:36,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/14b6a0db9d0d49b3baf28645590f017e is 50, key is test_row_0/B:col10/1732052555202/Put/seqid=0 2024-11-19T21:42:36,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742380_1556 (size=12001) 2024-11-19T21:42:36,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:36,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:36,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T21:42:36,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052616368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:36,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052616368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:36,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052616369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052616369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052616369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:36,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:36,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/14b6a0db9d0d49b3baf28645590f017e 2024-11-19T21:42:36,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/a29cec471bc04747884887db2fb3fcea is 50, key is test_row_0/C:col10/1732052555202/Put/seqid=0 2024-11-19T21:42:36,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742381_1557 (size=12001) 2024-11-19T21:42:36,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:36,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:36,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,820 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T21:42:36,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:36,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:36,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/a29cec471bc04747884887db2fb3fcea 2024-11-19T21:42:36,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/342ac102b61a4986bd84de2c1f237adc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/342ac102b61a4986bd84de2c1f237adc 2024-11-19T21:42:36,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/342ac102b61a4986bd84de2c1f237adc, entries=200, sequenceid=16, filesize=38.6 K 2024-11-19T21:42:36,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/14b6a0db9d0d49b3baf28645590f017e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/14b6a0db9d0d49b3baf28645590f017e 2024-11-19T21:42:36,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/14b6a0db9d0d49b3baf28645590f017e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T21:42:36,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/a29cec471bc04747884887db2fb3fcea as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a29cec471bc04747884887db2fb3fcea 2024-11-19T21:42:36,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a29cec471bc04747884887db2fb3fcea, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T21:42:36,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b95a52d5be5c4d97b13c59880275ce5 in 1699ms, sequenceid=16, compaction requested=false 2024-11-19T21:42:36,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:37,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T21:42:37,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:37,030 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:42:37,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119750117d495f94b6497de3f50f3b5d74e_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052555214/Put/seqid=0 2024-11-19T21:42:37,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742382_1558 (size=12154) 2024-11-19T21:42:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T21:42:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:37,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052617382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052617385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052617385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052617386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052617387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:37,441 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119750117d495f94b6497de3f50f3b5d74e_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119750117d495f94b6497de3f50f3b5d74e_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/ae78f1877c3b4c3db1c7fe967ad7cc94, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/ae78f1877c3b4c3db1c7fe967ad7cc94 is 175, key is test_row_0/A:col10/1732052555214/Put/seqid=0 2024-11-19T21:42:37,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742383_1559 (size=30955) 2024-11-19T21:42:37,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052617498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052617498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052617498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052617499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052617499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052617708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052617708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052617708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052617709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052617709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:37,850 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/ae78f1877c3b4c3db1c7fe967ad7cc94 2024-11-19T21:42:37,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/d07ac4ca2c2143aa8bafb2062841c858 is 50, key is test_row_0/B:col10/1732052555214/Put/seqid=0 2024-11-19T21:42:37,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742384_1560 (size=12001) 2024-11-19T21:42:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052618018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052618018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052618018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052618018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052618018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,259 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/d07ac4ca2c2143aa8bafb2062841c858 2024-11-19T21:42:38,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/8a090ebd7a6a4c94ba2c2b4c02d662a2 is 50, key is test_row_0/C:col10/1732052555214/Put/seqid=0 2024-11-19T21:42:38,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742385_1561 (size=12001) 2024-11-19T21:42:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052618528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052618528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052618528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052618528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052618528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:38,668 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/8a090ebd7a6a4c94ba2c2b4c02d662a2 2024-11-19T21:42:38,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/ae78f1877c3b4c3db1c7fe967ad7cc94 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/ae78f1877c3b4c3db1c7fe967ad7cc94 2024-11-19T21:42:38,673 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/ae78f1877c3b4c3db1c7fe967ad7cc94, entries=150, sequenceid=40, filesize=30.2 K 2024-11-19T21:42:38,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/d07ac4ca2c2143aa8bafb2062841c858 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d07ac4ca2c2143aa8bafb2062841c858 2024-11-19T21:42:38,676 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d07ac4ca2c2143aa8bafb2062841c858, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T21:42:38,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/8a090ebd7a6a4c94ba2c2b4c02d662a2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8a090ebd7a6a4c94ba2c2b4c02d662a2 2024-11-19T21:42:38,680 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8a090ebd7a6a4c94ba2c2b4c02d662a2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T21:42:38,683 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8b95a52d5be5c4d97b13c59880275ce5 in 1653ms, sequenceid=40, compaction requested=false 2024-11-19T21:42:38,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:38,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:38,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-19T21:42:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-19T21:42:38,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T21:42:38,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4830 sec 2024-11-19T21:42:38,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.4930 sec 2024-11-19T21:42:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T21:42:39,358 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:39,359 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-19T21:42:39,362 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T21:42:39,370 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:39,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T21:42:39,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T21:42:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:39,523 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:42:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b4e431581d364a4c9951efb532503b1f_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052557384/Put/seqid=0 2024-11-19T21:42:39,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742386_1562 (size=12154) 2024-11-19T21:42:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:39,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:39,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052619558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052619559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052619559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052619561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052619562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052619668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052619668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052619668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052619668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052619668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T21:42:39,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052619878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052619878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052619878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052619878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052619879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:39,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:39,935 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b4e431581d364a4c9951efb532503b1f_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b4e431581d364a4c9951efb532503b1f_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:39,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/34673efba659475e9b54f69feee55ca8, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:39,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/34673efba659475e9b54f69feee55ca8 is 175, key is test_row_0/A:col10/1732052557384/Put/seqid=0 2024-11-19T21:42:39,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742387_1563 (size=30955) 2024-11-19T21:42:39,945 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/34673efba659475e9b54f69feee55ca8 2024-11-19T21:42:39,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e2d4f8614943467ba215b8df167e7e05 is 50, key is test_row_0/B:col10/1732052557384/Put/seqid=0 2024-11-19T21:42:39,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742388_1564 (size=12001) 2024-11-19T21:42:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T21:42:40,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052620188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052620188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052620188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052620188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052620188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,353 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e2d4f8614943467ba215b8df167e7e05 2024-11-19T21:42:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/a49a94d4f23b4912ad33b1d79c22c7d4 is 50, key is test_row_0/C:col10/1732052557384/Put/seqid=0 2024-11-19T21:42:40,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742389_1565 (size=12001) 2024-11-19T21:42:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T21:42:40,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052620698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052620698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052620698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052620698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052620698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:40,762 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/a49a94d4f23b4912ad33b1d79c22c7d4 2024-11-19T21:42:40,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/34673efba659475e9b54f69feee55ca8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/34673efba659475e9b54f69feee55ca8 2024-11-19T21:42:40,767 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/34673efba659475e9b54f69feee55ca8, entries=150, sequenceid=52, filesize=30.2 K 2024-11-19T21:42:40,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e2d4f8614943467ba215b8df167e7e05 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e2d4f8614943467ba215b8df167e7e05 2024-11-19T21:42:40,772 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e2d4f8614943467ba215b8df167e7e05, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T21:42:40,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/a49a94d4f23b4912ad33b1d79c22c7d4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a49a94d4f23b4912ad33b1d79c22c7d4 2024-11-19T21:42:40,775 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a49a94d4f23b4912ad33b1d79c22c7d4, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T21:42:40,776 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b95a52d5be5c4d97b13c59880275ce5 in 1252ms, sequenceid=52, compaction requested=true 2024-11-19T21:42:40,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:40,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:40,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-19T21:42:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-19T21:42:40,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-19T21:42:40,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-11-19T21:42:40,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.4200 sec 2024-11-19T21:42:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T21:42:41,508 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:41,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-19T21:42:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T21:42:41,510 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:41,511 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:41,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T21:42:41,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T21:42:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:41,663 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:42:41,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e98e956c11e465d8bbbcf70b9085123_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052559561/Put/seqid=0 2024-11-19T21:42:41,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742390_1566 (size=12154) 2024-11-19T21:42:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:41,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052621715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052621716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052621716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:41,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052621718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:41,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052621718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:41,722 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T21:42:41,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:41,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052621828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052621828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052621829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052621829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052622038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052622038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052622038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052622038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:42,074 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e98e956c11e465d8bbbcf70b9085123_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e98e956c11e465d8bbbcf70b9085123_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:42,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/a6cc7e5790b64697b452017ff28b8aac, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:42,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/a6cc7e5790b64697b452017ff28b8aac is 175, key is test_row_0/A:col10/1732052559561/Put/seqid=0 2024-11-19T21:42:42,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742391_1567 (size=30955) 2024-11-19T21:42:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T21:42:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052622348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052622348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052622348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052622348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,478 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/a6cc7e5790b64697b452017ff28b8aac 2024-11-19T21:42:42,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/90ac60c4747d4b44b04a7c1dc73bcb23 is 50, key is test_row_0/B:col10/1732052559561/Put/seqid=0 2024-11-19T21:42:42,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742392_1568 (size=12001) 2024-11-19T21:42:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T21:42:42,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052622858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052622858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052622858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052622858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:42,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:42,887 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/90ac60c4747d4b44b04a7c1dc73bcb23 2024-11-19T21:42:42,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/6f819d824e8a4b39ac5077cefbd7f121 is 50, key is test_row_0/C:col10/1732052559561/Put/seqid=0 2024-11-19T21:42:42,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742393_1569 (size=12001) 2024-11-19T21:42:43,296 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/6f819d824e8a4b39ac5077cefbd7f121 2024-11-19T21:42:43,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/a6cc7e5790b64697b452017ff28b8aac as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a6cc7e5790b64697b452017ff28b8aac 2024-11-19T21:42:43,304 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a6cc7e5790b64697b452017ff28b8aac, entries=150, sequenceid=76, filesize=30.2 K 2024-11-19T21:42:43,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/90ac60c4747d4b44b04a7c1dc73bcb23 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/90ac60c4747d4b44b04a7c1dc73bcb23 2024-11-19T21:42:43,308 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/90ac60c4747d4b44b04a7c1dc73bcb23, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T21:42:43,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/6f819d824e8a4b39ac5077cefbd7f121 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6f819d824e8a4b39ac5077cefbd7f121 2024-11-19T21:42:43,315 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6f819d824e8a4b39ac5077cefbd7f121, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T21:42:43,315 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8b95a52d5be5c4d97b13c59880275ce5 in 1652ms, sequenceid=76, compaction requested=true 2024-11-19T21:42:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-19T21:42:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-19T21:42:43,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T21:42:43,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-11-19T21:42:43,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.8110 sec 2024-11-19T21:42:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T21:42:43,658 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:43,659 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-19T21:42:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T21:42:43,660 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:43,661 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:43,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:43,740 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:43,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:42:43,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198be69ae387314efea0527e9a4643c69a_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052561709/Put/seqid=0 2024-11-19T21:42:43,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742394_1570 (size=12154) 2024-11-19T21:42:43,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:43,752 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198be69ae387314efea0527e9a4643c69a_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198be69ae387314efea0527e9a4643c69a_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:43,752 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/e724dcdcbbad45f49a59602baa39882e, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:43,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/e724dcdcbbad45f49a59602baa39882e is 175, key is test_row_0/A:col10/1732052561709/Put/seqid=0 2024-11-19T21:42:43,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742395_1571 (size=30955) 2024-11-19T21:42:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T21:42:43,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T21:42:43,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:43,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:43,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:43,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:43,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052623834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:43,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052623868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:43,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052623868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:43,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:43,871 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052623868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:43,871 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,872 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:43,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052623878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:43,880 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:43,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052623948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:43,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T21:42:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:43,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T21:42:44,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T21:42:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/e724dcdcbbad45f49a59602baa39882e 2024-11-19T21:42:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052624158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:44,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e3488a0f18014000b603830c3ae24b9e is 50, key is test_row_0/B:col10/1732052561709/Put/seqid=0 2024-11-19T21:42:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:44,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742396_1572 (size=12001) 2024-11-19T21:42:44,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T21:42:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T21:42:44,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T21:42:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052624468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:44,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e3488a0f18014000b603830c3ae24b9e 2024-11-19T21:42:44,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/3f44cfe6e02440aab5004026d5691e71 is 50, key is test_row_0/C:col10/1732052561709/Put/seqid=0 2024-11-19T21:42:44,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742397_1573 (size=12001) 2024-11-19T21:42:44,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T21:42:44,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:44,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T21:42:44,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:44,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T21:42:44,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T21:42:44,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:44,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:44,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/3f44cfe6e02440aab5004026d5691e71 2024-11-19T21:42:44,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/e724dcdcbbad45f49a59602baa39882e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/e724dcdcbbad45f49a59602baa39882e 2024-11-19T21:42:44,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/e724dcdcbbad45f49a59602baa39882e, entries=150, sequenceid=89, filesize=30.2 K 2024-11-19T21:42:44,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e3488a0f18014000b603830c3ae24b9e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e3488a0f18014000b603830c3ae24b9e 2024-11-19T21:42:44,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e3488a0f18014000b603830c3ae24b9e, entries=150, sequenceid=89, filesize=11.7 K 2024-11-19T21:42:44,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/3f44cfe6e02440aab5004026d5691e71 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/3f44cfe6e02440aab5004026d5691e71 2024-11-19T21:42:44,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052624978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:44,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/3f44cfe6e02440aab5004026d5691e71, entries=150, sequenceid=89, filesize=11.7 K 2024-11-19T21:42:44,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8b95a52d5be5c4d97b13c59880275ce5 in 1245ms, sequenceid=89, compaction requested=true 2024-11-19T21:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:44,985 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T21:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:44,985 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T21:42:44,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:44,986 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T21:42:44,986 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163369 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T21:42:44,986 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/A is initiating minor compaction (all files) 2024-11-19T21:42:44,986 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/B is initiating minor compaction (all files) 2024-11-19T21:42:44,986 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/B in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,986 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/14b6a0db9d0d49b3baf28645590f017e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d07ac4ca2c2143aa8bafb2062841c858, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e2d4f8614943467ba215b8df167e7e05, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/90ac60c4747d4b44b04a7c1dc73bcb23, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e3488a0f18014000b603830c3ae24b9e] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=58.6 K 2024-11-19T21:42:44,986 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/A in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,986 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/342ac102b61a4986bd84de2c1f237adc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/ae78f1877c3b4c3db1c7fe967ad7cc94, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/34673efba659475e9b54f69feee55ca8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a6cc7e5790b64697b452017ff28b8aac, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/e724dcdcbbad45f49a59602baa39882e] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=159.5 K 2024-11-19T21:42:44,987 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:44,987 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/342ac102b61a4986bd84de2c1f237adc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/ae78f1877c3b4c3db1c7fe967ad7cc94, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/34673efba659475e9b54f69feee55ca8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a6cc7e5790b64697b452017ff28b8aac, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/e724dcdcbbad45f49a59602baa39882e] 2024-11-19T21:42:44,987 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 14b6a0db9d0d49b3baf28645590f017e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732052555202 2024-11-19T21:42:44,987 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 342ac102b61a4986bd84de2c1f237adc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732052555200 2024-11-19T21:42:44,987 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d07ac4ca2c2143aa8bafb2062841c858, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732052555214 2024-11-19T21:42:44,987 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d4f8614943467ba215b8df167e7e05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732052557381 2024-11-19T21:42:44,987 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae78f1877c3b4c3db1c7fe967ad7cc94, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732052555214 2024-11-19T21:42:44,987 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 90ac60c4747d4b44b04a7c1dc73bcb23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732052559558 2024-11-19T21:42:44,988 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34673efba659475e9b54f69feee55ca8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732052557381 2024-11-19T21:42:44,988 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6cc7e5790b64697b452017ff28b8aac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732052559558 2024-11-19T21:42:44,988 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e3488a0f18014000b603830c3ae24b9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732052561709 2024-11-19T21:42:44,988 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting e724dcdcbbad45f49a59602baa39882e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732052561709 2024-11-19T21:42:44,994 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:44,996 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:44,996 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111937f680837a094dda83b93b693ec8ac53_8b95a52d5be5c4d97b13c59880275ce5 store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:44,996 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/8fdc1292e92644378965db43440fd614 is 50, key is test_row_0/B:col10/1732052561709/Put/seqid=0 2024-11-19T21:42:44,999 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111937f680837a094dda83b93b693ec8ac53_8b95a52d5be5c4d97b13c59880275ce5, store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:44,999 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111937f680837a094dda83b93b693ec8ac53_8b95a52d5be5c4d97b13c59880275ce5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:45,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742398_1574 (size=12173) 2024-11-19T21:42:45,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742399_1575 (size=4469) 2024-11-19T21:42:45,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T21:42:45,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:45,033 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T21:42:45,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111919696c7816334d55a30169e699ae928b_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052563801/Put/seqid=0 2024-11-19T21:42:45,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742400_1576 (size=12154) 2024-11-19T21:42:45,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,043 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111919696c7816334d55a30169e699ae928b_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111919696c7816334d55a30169e699ae928b_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:45,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/de74b4a3ae9e4ca9882861926fbf5863, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/de74b4a3ae9e4ca9882861926fbf5863 is 175, key is test_row_0/A:col10/1732052563801/Put/seqid=0 2024-11-19T21:42:45,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742401_1577 (size=30955) 2024-11-19T21:42:45,404 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:45,404 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/8fdc1292e92644378965db43440fd614 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/8fdc1292e92644378965db43440fd614 2024-11-19T21:42:45,404 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/4194529ba33644dfb0d82a4f90637ca5 is 175, key is test_row_0/A:col10/1732052561709/Put/seqid=0 2024-11-19T21:42:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742402_1578 (size=31127) 2024-11-19T21:42:45,408 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/B of 8b95a52d5be5c4d97b13c59880275ce5 into 8fdc1292e92644378965db43440fd614(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:45,408 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:45,408 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/B, priority=11, startTime=1732052564985; duration=0sec 2024-11-19T21:42:45,408 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:45,408 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:B 2024-11-19T21:42:45,408 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T21:42:45,411 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T21:42:45,411 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/C is initiating minor compaction (all files) 2024-11-19T21:42:45,411 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/C in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:45,411 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a29cec471bc04747884887db2fb3fcea, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8a090ebd7a6a4c94ba2c2b4c02d662a2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a49a94d4f23b4912ad33b1d79c22c7d4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6f819d824e8a4b39ac5077cefbd7f121, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/3f44cfe6e02440aab5004026d5691e71] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=58.6 K 2024-11-19T21:42:45,412 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a29cec471bc04747884887db2fb3fcea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732052555202 2024-11-19T21:42:45,412 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/4194529ba33644dfb0d82a4f90637ca5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4194529ba33644dfb0d82a4f90637ca5 2024-11-19T21:42:45,412 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a090ebd7a6a4c94ba2c2b4c02d662a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732052555214 2024-11-19T21:42:45,412 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a49a94d4f23b4912ad33b1d79c22c7d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732052557381 2024-11-19T21:42:45,413 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f819d824e8a4b39ac5077cefbd7f121, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732052559558 2024-11-19T21:42:45,413 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f44cfe6e02440aab5004026d5691e71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732052561709 2024-11-19T21:42:45,416 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/A of 8b95a52d5be5c4d97b13c59880275ce5 into 4194529ba33644dfb0d82a4f90637ca5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:45,416 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:45,416 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/A, priority=11, startTime=1732052564985; duration=0sec 2024-11-19T21:42:45,416 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:45,416 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:A 2024-11-19T21:42:45,421 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#C#compaction#495 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:45,421 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/6d854027df404d419a55907b6c5250b1 is 50, key is test_row_0/C:col10/1732052561709/Put/seqid=0 2024-11-19T21:42:45,448 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/de74b4a3ae9e4ca9882861926fbf5863 2024-11-19T21:42:45,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742403_1579 (size=12173) 2024-11-19T21:42:45,452 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/6d854027df404d419a55907b6c5250b1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6d854027df404d419a55907b6c5250b1 2024-11-19T21:42:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/d8250395c351458b9ffd377ca4ed8de1 is 50, key is test_row_0/B:col10/1732052563801/Put/seqid=0 2024-11-19T21:42:45,457 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/C of 8b95a52d5be5c4d97b13c59880275ce5 into 6d854027df404d419a55907b6c5250b1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:45,457 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:45,457 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/C, priority=11, startTime=1732052564985; duration=0sec 2024-11-19T21:42:45,457 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:45,457 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:C 2024-11-19T21:42:45,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742404_1580 (size=12001) 2024-11-19T21:42:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T21:42:45,858 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/d8250395c351458b9ffd377ca4ed8de1 2024-11-19T21:42:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/92f284763a864407966296df3c96e05b is 50, key is test_row_0/C:col10/1732052563801/Put/seqid=0 2024-11-19T21:42:45,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742405_1581 (size=12001) 2024-11-19T21:42:45,872 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/92f284763a864407966296df3c96e05b 2024-11-19T21:42:45,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/de74b4a3ae9e4ca9882861926fbf5863 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/de74b4a3ae9e4ca9882861926fbf5863 2024-11-19T21:42:45,880 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/de74b4a3ae9e4ca9882861926fbf5863, entries=150, sequenceid=112, filesize=30.2 K 2024-11-19T21:42:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:45,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:45,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/d8250395c351458b9ffd377ca4ed8de1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d8250395c351458b9ffd377ca4ed8de1 2024-11-19T21:42:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,884 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d8250395c351458b9ffd377ca4ed8de1, entries=150, sequenceid=112, filesize=11.7 K 2024-11-19T21:42:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/92f284763a864407966296df3c96e05b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/92f284763a864407966296df3c96e05b 2024-11-19T21:42:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,887 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/92f284763a864407966296df3c96e05b, entries=150, sequenceid=112, filesize=11.7 K 2024-11-19T21:42:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,888 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=13.42 KB/13740 for 8b95a52d5be5c4d97b13c59880275ce5 in 855ms, sequenceid=112, compaction requested=false 2024-11-19T21:42:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:45,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:45,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-19T21:42:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-19T21:42:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-19T21:42:45,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2290 sec 2024-11-19T21:42:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.2330 sec 2024-11-19T21:42:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:45,905 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:45,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:42:45,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119be9f65e3b5f242f6a00b28ef0bde6948_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052565905/Put/seqid=0 2024-11-19T21:42:45,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742406_1582 (size=14594) 2024-11-19T21:42:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052625956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052625959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052625960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052625960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:45,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052625987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:45,989 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_2' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052626068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052626068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052626068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052626068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052626278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052626278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052626278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052626278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,314 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:46,317 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119be9f65e3b5f242f6a00b28ef0bde6948_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119be9f65e3b5f242f6a00b28ef0bde6948_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:46,317 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/b181184f474442afbe25524842bdd456, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:46,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/b181184f474442afbe25524842bdd456 is 175, key is test_row_0/A:col10/1732052565905/Put/seqid=0 2024-11-19T21:42:46,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742407_1583 (size=39549) 2024-11-19T21:42:46,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/b181184f474442afbe25524842bdd456 2024-11-19T21:42:46,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/7e4cc7d473da41c7bb8fa4369ce66a1e is 50, key is test_row_0/B:col10/1732052565905/Put/seqid=0 2024-11-19T21:42:46,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742408_1584 (size=12001) 2024-11-19T21:42:46,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052626587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052626588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052626588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052626588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:46,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/7e4cc7d473da41c7bb8fa4369ce66a1e 2024-11-19T21:42:46,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/eeae3065a46344ad9f175d322f7f1008 is 50, key is test_row_0/C:col10/1732052565905/Put/seqid=0 2024-11-19T21:42:46,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742409_1585 (size=12001) 2024-11-19T21:42:47,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052627097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:47,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052627098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052627098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052627098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:47,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/eeae3065a46344ad9f175d322f7f1008 2024-11-19T21:42:47,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/b181184f474442afbe25524842bdd456 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b181184f474442afbe25524842bdd456 2024-11-19T21:42:47,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b181184f474442afbe25524842bdd456, entries=200, sequenceid=127, filesize=38.6 K 2024-11-19T21:42:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/7e4cc7d473da41c7bb8fa4369ce66a1e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/7e4cc7d473da41c7bb8fa4369ce66a1e 2024-11-19T21:42:47,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/7e4cc7d473da41c7bb8fa4369ce66a1e, entries=150, sequenceid=127, filesize=11.7 K 2024-11-19T21:42:47,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/eeae3065a46344ad9f175d322f7f1008 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/eeae3065a46344ad9f175d322f7f1008 2024-11-19T21:42:47,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/eeae3065a46344ad9f175d322f7f1008, entries=150, sequenceid=127, filesize=11.7 K 2024-11-19T21:42:47,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8b95a52d5be5c4d97b13c59880275ce5 in 1252ms, sequenceid=127, compaction requested=true 2024-11-19T21:42:47,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:47,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:47,157 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:47,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:47,157 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:47,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:47,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:47,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:47,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:47,158 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:47,159 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/B is initiating minor compaction (all files) 2024-11-19T21:42:47,159 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/B in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:47,159 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/8fdc1292e92644378965db43440fd614, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d8250395c351458b9ffd377ca4ed8de1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/7e4cc7d473da41c7bb8fa4369ce66a1e] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=35.3 K 2024-11-19T21:42:47,159 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101631 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:47,159 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/A is initiating minor compaction (all files) 2024-11-19T21:42:47,159 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/A in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:47,159 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4194529ba33644dfb0d82a4f90637ca5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/de74b4a3ae9e4ca9882861926fbf5863, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b181184f474442afbe25524842bdd456] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=99.2 K 2024-11-19T21:42:47,159 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:47,159 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4194529ba33644dfb0d82a4f90637ca5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/de74b4a3ae9e4ca9882861926fbf5863, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b181184f474442afbe25524842bdd456] 2024-11-19T21:42:47,159 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4194529ba33644dfb0d82a4f90637ca5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732052561709 2024-11-19T21:42:47,159 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fdc1292e92644378965db43440fd614, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732052561709 2024-11-19T21:42:47,160 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting de74b4a3ae9e4ca9882861926fbf5863, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732052563801 2024-11-19T21:42:47,160 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d8250395c351458b9ffd377ca4ed8de1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732052563801 2024-11-19T21:42:47,160 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting b181184f474442afbe25524842bdd456, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732052565894 2024-11-19T21:42:47,160 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e4cc7d473da41c7bb8fa4369ce66a1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732052565904 2024-11-19T21:42:47,165 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:47,168 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:47,168 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/84c2b353d4a44a8789d9c854db01c57d is 50, key is test_row_0/B:col10/1732052565905/Put/seqid=0 2024-11-19T21:42:47,171 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195a864ab44b6f47de8e661dd25c273eee_8b95a52d5be5c4d97b13c59880275ce5 store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:47,172 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195a864ab44b6f47de8e661dd25c273eee_8b95a52d5be5c4d97b13c59880275ce5, store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:47,172 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195a864ab44b6f47de8e661dd25c273eee_8b95a52d5be5c4d97b13c59880275ce5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:47,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742410_1586 (size=12275) 2024-11-19T21:42:47,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742411_1587 (size=4469) 2024-11-19T21:42:47,598 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:47,598 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/84c2b353d4a44a8789d9c854db01c57d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/84c2b353d4a44a8789d9c854db01c57d 2024-11-19T21:42:47,598 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/988a29ad71c14ee4b54571b6175e7644 is 175, key is test_row_0/A:col10/1732052565905/Put/seqid=0 2024-11-19T21:42:47,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742412_1588 (size=31229) 2024-11-19T21:42:47,603 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/B of 8b95a52d5be5c4d97b13c59880275ce5 into 84c2b353d4a44a8789d9c854db01c57d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:47,603 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:47,603 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/B, priority=13, startTime=1732052567157; duration=0sec 2024-11-19T21:42:47,604 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:47,604 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:B 2024-11-19T21:42:47,604 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:47,607 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:47,607 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/C is initiating minor compaction (all files) 2024-11-19T21:42:47,607 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/C in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:47,607 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6d854027df404d419a55907b6c5250b1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/92f284763a864407966296df3c96e05b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/eeae3065a46344ad9f175d322f7f1008] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=35.3 K 2024-11-19T21:42:47,607 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d854027df404d419a55907b6c5250b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732052561709 2024-11-19T21:42:47,608 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/988a29ad71c14ee4b54571b6175e7644 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/988a29ad71c14ee4b54571b6175e7644 2024-11-19T21:42:47,608 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f284763a864407966296df3c96e05b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732052563801 2024-11-19T21:42:47,609 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting eeae3065a46344ad9f175d322f7f1008, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732052565904 2024-11-19T21:42:47,612 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/A of 8b95a52d5be5c4d97b13c59880275ce5 into 988a29ad71c14ee4b54571b6175e7644(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:47,612 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:47,612 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/A, priority=13, startTime=1732052567157; duration=0sec 2024-11-19T21:42:47,612 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:47,612 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:A 2024-11-19T21:42:47,616 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:47,616 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/d159cfaa69f64078b3c2ab04751ec39e is 50, key is test_row_0/C:col10/1732052565905/Put/seqid=0 2024-11-19T21:42:47,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742413_1589 (size=12275) 2024-11-19T21:42:47,622 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/d159cfaa69f64078b3c2ab04751ec39e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d159cfaa69f64078b3c2ab04751ec39e 2024-11-19T21:42:47,626 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/C of 8b95a52d5be5c4d97b13c59880275ce5 into d159cfaa69f64078b3c2ab04751ec39e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:47,626 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:47,626 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/C, priority=13, startTime=1732052567157; duration=0sec 2024-11-19T21:42:47,626 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:47,626 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:C 2024-11-19T21:42:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T21:42:47,828 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:47,828 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-19T21:42:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T21:42:47,830 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:47,831 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:47,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T21:42:47,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T21:42:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:47,983 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:42:47,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195ba1c3790bb24ba79e20baf72f82ab1a_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052565958/Put/seqid=0 2024-11-19T21:42:47,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742414_1590 (size=12304) 2024-11-19T21:42:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:48,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:48,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052628037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:48,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052628108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:48,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052628108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:48,110 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:48,111 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:48,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052628118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:48,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052628118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:48,120 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:48,120 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T21:42:48,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052628148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:48,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:48,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052628358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:48,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:48,396 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195ba1c3790bb24ba79e20baf72f82ab1a_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ba1c3790bb24ba79e20baf72f82ab1a_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:48,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/b6b5964eee9a4488b067e4ffb37bd119, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:48,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/b6b5964eee9a4488b067e4ffb37bd119 is 175, key is test_row_0/A:col10/1732052565958/Put/seqid=0 2024-11-19T21:42:48,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742415_1591 (size=31105) 2024-11-19T21:42:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T21:42:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052628668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:48,812 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/b6b5964eee9a4488b067e4ffb37bd119 2024-11-19T21:42:48,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e91dd5ee818a4aaaa3647dac4100f334 is 50, key is test_row_0/B:col10/1732052565958/Put/seqid=0 2024-11-19T21:42:48,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742416_1592 (size=12151) 2024-11-19T21:42:48,821 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e91dd5ee818a4aaaa3647dac4100f334 2024-11-19T21:42:48,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/8e71cea084fa4c6e86cc49dedba871a4 is 50, key is test_row_0/C:col10/1732052565958/Put/seqid=0 2024-11-19T21:42:48,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742417_1593 (size=12151) 2024-11-19T21:42:48,832 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/8e71cea084fa4c6e86cc49dedba871a4 2024-11-19T21:42:48,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/b6b5964eee9a4488b067e4ffb37bd119 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b6b5964eee9a4488b067e4ffb37bd119 2024-11-19T21:42:48,840 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b6b5964eee9a4488b067e4ffb37bd119, entries=150, sequenceid=155, filesize=30.4 K 2024-11-19T21:42:48,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e91dd5ee818a4aaaa3647dac4100f334 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e91dd5ee818a4aaaa3647dac4100f334 2024-11-19T21:42:48,846 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e91dd5ee818a4aaaa3647dac4100f334, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T21:42:48,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/8e71cea084fa4c6e86cc49dedba871a4 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8e71cea084fa4c6e86cc49dedba871a4 2024-11-19T21:42:48,850 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8e71cea084fa4c6e86cc49dedba871a4, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T21:42:48,851 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8b95a52d5be5c4d97b13c59880275ce5 in 868ms, sequenceid=155, compaction requested=false 2024-11-19T21:42:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-19T21:42:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-19T21:42:48,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T21:42:48,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0220 sec 2024-11-19T21:42:48,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0280 sec 2024-11-19T21:42:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T21:42:48,969 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:48,969 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-19T21:42:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T21:42:48,970 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:48,971 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:48,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T21:42:49,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T21:42:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:49,123 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T21:42:49,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193b54373fd46542e2912baada605d2fe2_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052568035/Put/seqid=0 2024-11-19T21:42:49,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742418_1594 (size=12304) 2024-11-19T21:42:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:49,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:49,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052629271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:49,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T21:42:49,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:49,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052629378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:49,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:49,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:49,540 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193b54373fd46542e2912baada605d2fe2_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193b54373fd46542e2912baada605d2fe2_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/4998537b9ad9492db03540375f013343, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:49,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/4998537b9ad9492db03540375f013343 is 175, key is test_row_0/A:col10/1732052568035/Put/seqid=0 2024-11-19T21:42:49,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742419_1595 (size=31105) 2024-11-19T21:42:49,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052629588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T21:42:49,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052629898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:49,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:49,952 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/4998537b9ad9492db03540375f013343 2024-11-19T21:42:49,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/3d6c1180ace949de9b00101ee2aac079 is 50, key is test_row_0/B:col10/1732052568035/Put/seqid=0 2024-11-19T21:42:49,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742420_1596 (size=12151) 2024-11-19T21:42:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T21:42:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052630128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:50,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052630128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:50,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052630129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:50,131 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:50,131 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:50,132 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:50,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052630149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:50,152 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:50,365 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/3d6c1180ace949de9b00101ee2aac079 2024-11-19T21:42:50,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/27fe02b455504662a41fe1c1efab3714 is 50, key is test_row_0/C:col10/1732052568035/Put/seqid=0 2024-11-19T21:42:50,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742421_1597 (size=12151) 2024-11-19T21:42:50,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:50,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052630408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:50,775 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/27fe02b455504662a41fe1c1efab3714 2024-11-19T21:42:50,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/4998537b9ad9492db03540375f013343 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4998537b9ad9492db03540375f013343 2024-11-19T21:42:50,781 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4998537b9ad9492db03540375f013343, entries=150, sequenceid=166, filesize=30.4 K 2024-11-19T21:42:50,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/3d6c1180ace949de9b00101ee2aac079 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/3d6c1180ace949de9b00101ee2aac079 2024-11-19T21:42:50,784 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/3d6c1180ace949de9b00101ee2aac079, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T21:42:50,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/27fe02b455504662a41fe1c1efab3714 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/27fe02b455504662a41fe1c1efab3714 2024-11-19T21:42:50,788 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/27fe02b455504662a41fe1c1efab3714, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T21:42:50,789 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8b95a52d5be5c4d97b13c59880275ce5 in 1666ms, sequenceid=166, compaction requested=true 2024-11-19T21:42:50,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:50,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:50,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-19T21:42:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-19T21:42:50,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T21:42:50,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8190 sec 2024-11-19T21:42:50,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8230 sec 2024-11-19T21:42:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T21:42:51,118 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:51,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-19T21:42:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T21:42:51,120 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:51,121 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:51,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T21:42:51,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T21:42:51,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:51,273 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T21:42:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b3d4ef714002491ab950ad9a6759d95c_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052569242/Put/seqid=0 2024-11-19T21:42:51,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742422_1598 (size=12304) 2024-11-19T21:42:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:51,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T21:42:51,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052631457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052631568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:51,686 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b3d4ef714002491ab950ad9a6759d95c_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b3d4ef714002491ab950ad9a6759d95c_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:51,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/5e9dfe67ff2d49a89e1b5195c3ea793c, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/5e9dfe67ff2d49a89e1b5195c3ea793c is 175, key is test_row_0/A:col10/1732052569242/Put/seqid=0 2024-11-19T21:42:51,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742423_1599 (size=31105) 2024-11-19T21:42:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T21:42:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052631778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:52,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052632088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:52,091 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/5e9dfe67ff2d49a89e1b5195c3ea793c 2024-11-19T21:42:52,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/66b474a10df7441da65ef001f6473117 is 50, key is test_row_0/B:col10/1732052569242/Put/seqid=0 2024-11-19T21:42:52,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742424_1600 (size=12151) 2024-11-19T21:42:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T21:42:52,277 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T21:42:52,504 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/66b474a10df7441da65ef001f6473117 2024-11-19T21:42:52,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/bf5a40fd0a16484f89f71a5def0a0753 is 50, key is test_row_0/C:col10/1732052569242/Put/seqid=0 2024-11-19T21:42:52,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742425_1601 (size=12151) 2024-11-19T21:42:52,514 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/bf5a40fd0a16484f89f71a5def0a0753 2024-11-19T21:42:52,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/5e9dfe67ff2d49a89e1b5195c3ea793c as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/5e9dfe67ff2d49a89e1b5195c3ea793c 2024-11-19T21:42:52,520 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/5e9dfe67ff2d49a89e1b5195c3ea793c, entries=150, sequenceid=191, filesize=30.4 K 2024-11-19T21:42:52,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/66b474a10df7441da65ef001f6473117 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/66b474a10df7441da65ef001f6473117 2024-11-19T21:42:52,523 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/66b474a10df7441da65ef001f6473117, entries=150, sequenceid=191, filesize=11.9 K 2024-11-19T21:42:52,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/bf5a40fd0a16484f89f71a5def0a0753 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/bf5a40fd0a16484f89f71a5def0a0753 2024-11-19T21:42:52,526 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/bf5a40fd0a16484f89f71a5def0a0753, entries=150, sequenceid=191, filesize=11.9 K 2024-11-19T21:42:52,527 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8b95a52d5be5c4d97b13c59880275ce5 in 1254ms, sequenceid=191, compaction requested=true 2024-11-19T21:42:52,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:52,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:52,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-19T21:42:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-19T21:42:52,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T21:42:52,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-11-19T21:42:52,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4110 sec 2024-11-19T21:42:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:52,601 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:52,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T21:42:52,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194378c35f69984fb690f1bbac34d40ade_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052571440/Put/seqid=0 2024-11-19T21:42:52,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742426_1602 (size=12304) 2024-11-19T21:42:52,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052632696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052632808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:53,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:53,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194378c35f69984fb690f1bbac34d40ade_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194378c35f69984fb690f1bbac34d40ade_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:53,013 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/69398b0e5338406184bb88e1756b86ab, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:53,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/69398b0e5338406184bb88e1756b86ab is 175, key is test_row_0/A:col10/1732052571440/Put/seqid=0 2024-11-19T21:42:53,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742427_1603 (size=31105) 2024-11-19T21:42:53,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052633018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T21:42:53,268 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:53,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T21:42:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-19T21:42:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T21:42:53,270 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T21:42:53,270 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T21:42:53,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T21:42:53,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052633328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T21:42:53,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/69398b0e5338406184bb88e1756b86ab 2024-11-19T21:42:53,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e64f7034d60648f5acf2fb52f09fb4f0 is 50, key is test_row_0/B:col10/1732052571440/Put/seqid=0 2024-11-19T21:42:53,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T21:42:53,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:53,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:53,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:53,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:53,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T21:42:53,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742428_1604 (size=12151) 2024-11-19T21:42:53,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e64f7034d60648f5acf2fb52f09fb4f0 2024-11-19T21:42:53,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/d16e102b930d4de0a0a804af7950a594 is 50, key is test_row_0/C:col10/1732052571440/Put/seqid=0 2024-11-19T21:42:53,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742429_1605 (size=12151) 2024-11-19T21:42:53,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/d16e102b930d4de0a0a804af7950a594 2024-11-19T21:42:53,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/69398b0e5338406184bb88e1756b86ab as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/69398b0e5338406184bb88e1756b86ab 2024-11-19T21:42:53,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/69398b0e5338406184bb88e1756b86ab, entries=150, sequenceid=203, filesize=30.4 K 2024-11-19T21:42:53,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e64f7034d60648f5acf2fb52f09fb4f0 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e64f7034d60648f5acf2fb52f09fb4f0 2024-11-19T21:42:53,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e64f7034d60648f5acf2fb52f09fb4f0, entries=150, sequenceid=203, filesize=11.9 K 2024-11-19T21:42:53,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/d16e102b930d4de0a0a804af7950a594 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d16e102b930d4de0a0a804af7950a594 2024-11-19T21:42:53,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d16e102b930d4de0a0a804af7950a594, entries=150, sequenceid=203, filesize=11.9 K 2024-11-19T21:42:53,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b95a52d5be5c4d97b13c59880275ce5 in 847ms, sequenceid=203, compaction requested=true 2024-11-19T21:42:53,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:53,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:53,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:53,449 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T21:42:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:53,449 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T21:42:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:53,450 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 155649 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T21:42:53,450 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60879 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T21:42:53,450 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/A is initiating minor compaction (all files) 2024-11-19T21:42:53,450 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/B is initiating minor compaction (all files) 2024-11-19T21:42:53,450 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/A in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:53,450 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/B in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:53,450 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/988a29ad71c14ee4b54571b6175e7644, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b6b5964eee9a4488b067e4ffb37bd119, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4998537b9ad9492db03540375f013343, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/5e9dfe67ff2d49a89e1b5195c3ea793c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/69398b0e5338406184bb88e1756b86ab] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=152.0 K 2024-11-19T21:42:53,450 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/84c2b353d4a44a8789d9c854db01c57d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e91dd5ee818a4aaaa3647dac4100f334, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/3d6c1180ace949de9b00101ee2aac079, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/66b474a10df7441da65ef001f6473117, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e64f7034d60648f5acf2fb52f09fb4f0] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=59.5 K 2024-11-19T21:42:53,450 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:53,450 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/988a29ad71c14ee4b54571b6175e7644, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b6b5964eee9a4488b067e4ffb37bd119, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4998537b9ad9492db03540375f013343, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/5e9dfe67ff2d49a89e1b5195c3ea793c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/69398b0e5338406184bb88e1756b86ab] 2024-11-19T21:42:53,450 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 988a29ad71c14ee4b54571b6175e7644, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732052565904 2024-11-19T21:42:53,450 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 84c2b353d4a44a8789d9c854db01c57d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732052565904 2024-11-19T21:42:53,450 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6b5964eee9a4488b067e4ffb37bd119, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732052565954 2024-11-19T21:42:53,450 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e91dd5ee818a4aaaa3647dac4100f334, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732052565954 2024-11-19T21:42:53,451 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4998537b9ad9492db03540375f013343, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732052568013 2024-11-19T21:42:53,451 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d6c1180ace949de9b00101ee2aac079, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732052568013 2024-11-19T21:42:53,451 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e9dfe67ff2d49a89e1b5195c3ea793c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732052569242 2024-11-19T21:42:53,451 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 66b474a10df7441da65ef001f6473117, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732052569242 2024-11-19T21:42:53,451 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69398b0e5338406184bb88e1756b86ab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732052571440 2024-11-19T21:42:53,451 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting e64f7034d60648f5acf2fb52f09fb4f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732052571440 2024-11-19T21:42:53,458 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:53,461 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#B#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:53,461 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/184ba5bc253b475893fb65158a945aa7 is 50, key is test_row_0/B:col10/1732052571440/Put/seqid=0 2024-11-19T21:42:53,467 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119b09f4734db4f4778ac9f44d98e91dc15_8b95a52d5be5c4d97b13c59880275ce5 store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:53,469 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119b09f4734db4f4778ac9f44d98e91dc15_8b95a52d5be5c4d97b13c59880275ce5, store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:53,469 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b09f4734db4f4778ac9f44d98e91dc15_8b95a52d5be5c4d97b13c59880275ce5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:53,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742430_1606 (size=12595) 2024-11-19T21:42:53,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742431_1607 (size=4469) 2024-11-19T21:42:53,473 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#A#compaction#516 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:53,473 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/3f205c02f5a147cfbf6b5091c9e7ea46 is 175, key is test_row_0/A:col10/1732052571440/Put/seqid=0 2024-11-19T21:42:53,475 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/184ba5bc253b475893fb65158a945aa7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/184ba5bc253b475893fb65158a945aa7 2024-11-19T21:42:53,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742432_1608 (size=31549) 2024-11-19T21:42:53,479 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/3f205c02f5a147cfbf6b5091c9e7ea46 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3f205c02f5a147cfbf6b5091c9e7ea46 2024-11-19T21:42:53,479 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/B of 8b95a52d5be5c4d97b13c59880275ce5 into 184ba5bc253b475893fb65158a945aa7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:53,479 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:53,479 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/B, priority=11, startTime=1732052573449; duration=0sec 2024-11-19T21:42:53,480 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:53,480 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:B 2024-11-19T21:42:53,480 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T21:42:53,481 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60879 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T21:42:53,481 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/C is initiating minor compaction (all files) 2024-11-19T21:42:53,481 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/C in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:53,481 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d159cfaa69f64078b3c2ab04751ec39e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8e71cea084fa4c6e86cc49dedba871a4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/27fe02b455504662a41fe1c1efab3714, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/bf5a40fd0a16484f89f71a5def0a0753, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d16e102b930d4de0a0a804af7950a594] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=59.5 K 2024-11-19T21:42:53,481 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d159cfaa69f64078b3c2ab04751ec39e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732052565904 2024-11-19T21:42:53,481 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e71cea084fa4c6e86cc49dedba871a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732052565954 2024-11-19T21:42:53,482 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 27fe02b455504662a41fe1c1efab3714, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732052568013 2024-11-19T21:42:53,482 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5a40fd0a16484f89f71a5def0a0753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732052569242 2024-11-19T21:42:53,483 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting d16e102b930d4de0a0a804af7950a594, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732052571440 2024-11-19T21:42:53,484 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/A of 8b95a52d5be5c4d97b13c59880275ce5 into 3f205c02f5a147cfbf6b5091c9e7ea46(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:53,484 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:53,484 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/A, priority=11, startTime=1732052573448; duration=0sec 2024-11-19T21:42:53,484 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:53,484 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:A 2024-11-19T21:42:53,494 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:53,494 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/b4a9f03d5b4d40c4a5a847f11f7c21fc is 50, key is test_row_0/C:col10/1732052571440/Put/seqid=0 2024-11-19T21:42:53,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742433_1609 (size=12595) 2024-11-19T21:42:53,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42821 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T21:42:53,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:53,576 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:42:53,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194298ba069ed44a9a88a21ca3a6330fc8_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052572616/Put/seqid=0 2024-11-19T21:42:53,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742434_1610 (size=12304) 2024-11-19T21:42:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T21:42:53,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. as already flushing 2024-11-19T21:42:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:53,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052633862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T21:42:53,900 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/b4a9f03d5b4d40c4a5a847f11f7c21fc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/b4a9f03d5b4d40c4a5a847f11f7c21fc 2024-11-19T21:42:53,903 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/C of 8b95a52d5be5c4d97b13c59880275ce5 into b4a9f03d5b4d40c4a5a847f11f7c21fc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:53,903 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:53,903 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/C, priority=11, startTime=1732052573449; duration=0sec 2024-11-19T21:42:53,904 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:53,904 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:C 2024-11-19T21:42:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052633968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:53,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:53,988 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194298ba069ed44a9a88a21ca3a6330fc8_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194298ba069ed44a9a88a21ca3a6330fc8_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/3b390c8a226b4fc1863a6ca6b10a0ade, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/3b390c8a226b4fc1863a6ca6b10a0ade is 175, key is test_row_0/A:col10/1732052572616/Put/seqid=0 2024-11-19T21:42:53,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742435_1611 (size=31105) 2024-11-19T21:42:53,993 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/3b390c8a226b4fc1863a6ca6b10a0ade 2024-11-19T21:42:53,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/bb3e21bc1d5e4d319a4ba26d8454cc06 is 50, key is test_row_0/B:col10/1732052572616/Put/seqid=0 2024-11-19T21:42:54,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742436_1612 (size=12151) 2024-11-19T21:42:54,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43462 deadline: 1732052634148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:54,150 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:54,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43438 deadline: 1732052634168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:54,171 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:54,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732052634178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:54,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052634178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:54,180 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_0' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:54,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1732052634188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:54,190 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 707262ffa86d,42821,1732052394005 for 'test_row_1' in 8b95a52d5be5c4d97b13c59880275ce5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T21:42:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:54,401 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/bb3e21bc1d5e4d319a4ba26d8454cc06 2024-11-19T21:42:54,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/adfa111661e140a486f1c95996e9050e is 50, key is test_row_0/C:col10/1732052572616/Put/seqid=0 2024-11-19T21:42:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T21:42:54,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742437_1613 (size=12151) 2024-11-19T21:42:54,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052634488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:54,810 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/adfa111661e140a486f1c95996e9050e 2024-11-19T21:42:54,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/3b390c8a226b4fc1863a6ca6b10a0ade as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3b390c8a226b4fc1863a6ca6b10a0ade 2024-11-19T21:42:54,816 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3b390c8a226b4fc1863a6ca6b10a0ade, entries=150, sequenceid=229, filesize=30.4 K 2024-11-19T21:42:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/bb3e21bc1d5e4d319a4ba26d8454cc06 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/bb3e21bc1d5e4d319a4ba26d8454cc06 2024-11-19T21:42:54,819 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/bb3e21bc1d5e4d319a4ba26d8454cc06, entries=150, sequenceid=229, filesize=11.9 K 2024-11-19T21:42:54,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/adfa111661e140a486f1c95996e9050e as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/adfa111661e140a486f1c95996e9050e 2024-11-19T21:42:54,822 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/adfa111661e140a486f1c95996e9050e, entries=150, sequenceid=229, filesize=11.9 K 2024-11-19T21:42:54,823 INFO [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8b95a52d5be5c4d97b13c59880275ce5 in 1247ms, sequenceid=229, compaction requested=false 2024-11-19T21:42:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:54,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/707262ffa86d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-19T21:42:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-19T21:42:54,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T21:42:54,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5530 sec 2024-11-19T21:42:54,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.5580 sec 2024-11-19T21:42:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:54,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T21:42:55,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111926b0e2e0e891477284a821c64bbbd20d_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052573858/Put/seqid=0 2024-11-19T21:42:55,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742438_1614 (size=14794) 2024-11-19T21:42:55,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052635076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:55,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052635188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:55,195 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-19T21:42:55,195 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:55,195 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,195 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:55,195 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-19T21:42:55,195 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:55,195 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,195 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:55,197 INFO [Thread-2479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2479. 2024-11-19T21:42:55,197 DEBUG [Thread-2479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:55,197 DEBUG [Thread-2479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,197 DEBUG [Thread-2479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,197 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:55,198 INFO [Thread-2477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2477. 2024-11-19T21:42:55,198 DEBUG [Thread-2477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:55,198 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,198 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:55,199 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-19T21:42:55,199 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:55,199 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,199 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:55,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:55,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052635399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:55,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:55,410 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111926b0e2e0e891477284a821c64bbbd20d_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111926b0e2e0e891477284a821c64bbbd20d_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:55,411 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/d156df28647145d0842079056608249b, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:55,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/d156df28647145d0842079056608249b is 175, key is test_row_0/A:col10/1732052573858/Put/seqid=0 2024-11-19T21:42:55,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742439_1615 (size=39749) 2024-11-19T21:42:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T21:42:55,418 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:42:55,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T21:42:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43388 deadline: 1732052635709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 2024-11-19T21:42:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T21:42:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., hostname=707262ffa86d,42821,1732052394005, seqNum=5 because the exception is null or not the one we care about 2024-11-19T21:42:55,816 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/d156df28647145d0842079056608249b 2024-11-19T21:42:55,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/a97544fbd5cc4129ad35a5247cabd74d is 50, key is test_row_0/B:col10/1732052573858/Put/seqid=0 2024-11-19T21:42:55,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742440_1616 (size=12151) 2024-11-19T21:42:55,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/a97544fbd5cc4129ad35a5247cabd74d 2024-11-19T21:42:55,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/18a790d0356d4088a2dbe9f0bc23d7a3 is 50, key is test_row_0/C:col10/1732052573858/Put/seqid=0 2024-11-19T21:42:55,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742441_1617 (size=12151) 2024-11-19T21:42:55,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/18a790d0356d4088a2dbe9f0bc23d7a3 2024-11-19T21:42:55,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/d156df28647145d0842079056608249b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/d156df28647145d0842079056608249b 2024-11-19T21:42:55,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/d156df28647145d0842079056608249b, entries=200, sequenceid=243, filesize=38.8 K 2024-11-19T21:42:55,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/a97544fbd5cc4129ad35a5247cabd74d as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/a97544fbd5cc4129ad35a5247cabd74d 2024-11-19T21:42:55,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/a97544fbd5cc4129ad35a5247cabd74d, entries=150, sequenceid=243, filesize=11.9 K 2024-11-19T21:42:55,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/18a790d0356d4088a2dbe9f0bc23d7a3 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/18a790d0356d4088a2dbe9f0bc23d7a3 2024-11-19T21:42:55,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/18a790d0356d4088a2dbe9f0bc23d7a3, entries=150, sequenceid=243, filesize=11.9 K 2024-11-19T21:42:55,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8b95a52d5be5c4d97b13c59880275ce5 in 858ms, sequenceid=243, compaction requested=true 2024-11-19T21:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T21:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T21:42:55,857 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b95a52d5be5c4d97b13c59880275ce5:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T21:42:55,857 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:55,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:55,858 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:55,858 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:55,858 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/A is initiating minor compaction (all files) 2024-11-19T21:42:55,858 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/B is initiating minor compaction (all files) 2024-11-19T21:42:55,858 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/A in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:55,858 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/B in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:55,858 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3f205c02f5a147cfbf6b5091c9e7ea46, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3b390c8a226b4fc1863a6ca6b10a0ade, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/d156df28647145d0842079056608249b] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=100.0 K 2024-11-19T21:42:55,858 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/184ba5bc253b475893fb65158a945aa7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/bb3e21bc1d5e4d319a4ba26d8454cc06, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/a97544fbd5cc4129ad35a5247cabd74d] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=36.0 K 2024-11-19T21:42:55,858 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:55,858 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. files: [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3f205c02f5a147cfbf6b5091c9e7ea46, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3b390c8a226b4fc1863a6ca6b10a0ade, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/d156df28647145d0842079056608249b] 2024-11-19T21:42:55,858 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 184ba5bc253b475893fb65158a945aa7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732052571440 2024-11-19T21:42:55,858 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f205c02f5a147cfbf6b5091c9e7ea46, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732052571440 2024-11-19T21:42:55,859 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting bb3e21bc1d5e4d319a4ba26d8454cc06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732052572616 2024-11-19T21:42:55,859 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b390c8a226b4fc1863a6ca6b10a0ade, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732052572616 2024-11-19T21:42:55,859 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] compactions.Compactor(225): Compacting d156df28647145d0842079056608249b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732052573853 2024-11-19T21:42:55,859 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting a97544fbd5cc4129ad35a5247cabd74d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732052573858 2024-11-19T21:42:55,862 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:55,863 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#B#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:55,864 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/b057b42faa9a4beeab699fd346ce29a2 is 50, key is test_row_0/B:col10/1732052573858/Put/seqid=0 2024-11-19T21:42:55,864 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111983bc23310d5f4ff5bdb096c5987db527_8b95a52d5be5c4d97b13c59880275ce5 store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:55,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742442_1618 (size=12697) 2024-11-19T21:42:55,867 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111983bc23310d5f4ff5bdb096c5987db527_8b95a52d5be5c4d97b13c59880275ce5, store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:55,867 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111983bc23310d5f4ff5bdb096c5987db527_8b95a52d5be5c4d97b13c59880275ce5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:55,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742443_1619 (size=4469) 2024-11-19T21:42:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42821 {}] regionserver.HRegion(8855): Flush requested on 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:56,224 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T21:42:56,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T21:42:56,224 INFO [Thread-2462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2462. 2024-11-19T21:42:56,225 DEBUG [Thread-2462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:42:56,225 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:56,225 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:42:56,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:42:56,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111921afb220e32745a0ba70a52ec798756f_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052576220/Put/seqid=0 2024-11-19T21:42:56,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742444_1620 (size=12454) 2024-11-19T21:42:56,270 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#A#compaction#526 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:56,270 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/b057b42faa9a4beeab699fd346ce29a2 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/b057b42faa9a4beeab699fd346ce29a2 2024-11-19T21:42:56,270 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/a966a67be54e4d29aced0038e2c64773 is 175, key is test_row_0/A:col10/1732052573858/Put/seqid=0 2024-11-19T21:42:56,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742445_1621 (size=31651) 2024-11-19T21:42:56,274 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/B of 8b95a52d5be5c4d97b13c59880275ce5 into b057b42faa9a4beeab699fd346ce29a2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:56,274 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:56,274 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/B, priority=13, startTime=1732052575857; duration=0sec 2024-11-19T21:42:56,274 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T21:42:56,274 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:B 2024-11-19T21:42:56,274 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T21:42:56,274 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T21:42:56,275 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1541): 8b95a52d5be5c4d97b13c59880275ce5/C is initiating minor compaction (all files) 2024-11-19T21:42:56,275 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b95a52d5be5c4d97b13c59880275ce5/C in TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:42:56,275 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/b4a9f03d5b4d40c4a5a847f11f7c21fc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/adfa111661e140a486f1c95996e9050e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/18a790d0356d4088a2dbe9f0bc23d7a3] into tmpdir=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp, totalSize=36.0 K 2024-11-19T21:42:56,275 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting b4a9f03d5b4d40c4a5a847f11f7c21fc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732052571440 2024-11-19T21:42:56,275 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting adfa111661e140a486f1c95996e9050e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732052572616 2024-11-19T21:42:56,275 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a790d0356d4088a2dbe9f0bc23d7a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732052573858 2024-11-19T21:42:56,282 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b95a52d5be5c4d97b13c59880275ce5#C#compaction#528 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T21:42:56,282 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/f799e5fcf0f846e59483f0bde92578bf is 50, key is test_row_0/C:col10/1732052573858/Put/seqid=0 2024-11-19T21:42:56,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742446_1622 (size=12697) 2024-11-19T21:42:56,637 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:42:56,688 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111921afb220e32745a0ba70a52ec798756f_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111921afb220e32745a0ba70a52ec798756f_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:42:56,689 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/29de3a29b5fd4c2389b3819688694aa6, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:42:56,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/29de3a29b5fd4c2389b3819688694aa6 is 175, key is test_row_0/A:col10/1732052576220/Put/seqid=0 2024-11-19T21:42:56,692 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/a966a67be54e4d29aced0038e2c64773 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a966a67be54e4d29aced0038e2c64773 2024-11-19T21:42:56,692 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/f799e5fcf0f846e59483f0bde92578bf as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/f799e5fcf0f846e59483f0bde92578bf 2024-11-19T21:42:56,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742447_1623 (size=31255) 2024-11-19T21:42:56,698 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/C of 8b95a52d5be5c4d97b13c59880275ce5 into f799e5fcf0f846e59483f0bde92578bf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:56,698 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b95a52d5be5c4d97b13c59880275ce5/A of 8b95a52d5be5c4d97b13c59880275ce5 into a966a67be54e4d29aced0038e2c64773(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T21:42:56,698 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:56,698 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:42:56,698 INFO [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/C, priority=13, startTime=1732052575857; duration=0sec 2024-11-19T21:42:56,698 INFO [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., storeName=8b95a52d5be5c4d97b13c59880275ce5/A, priority=13, startTime=1732052575857; duration=0sec 2024-11-19T21:42:56,698 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:56,698 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T21:42:56,698 DEBUG [RS:0;707262ffa86d:42821-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:A 2024-11-19T21:42:56,698 DEBUG [RS:0;707262ffa86d:42821-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b95a52d5be5c4d97b13c59880275ce5:C 2024-11-19T21:42:57,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/29de3a29b5fd4c2389b3819688694aa6 2024-11-19T21:42:57,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/8321639ed06d4bb2bc83498f9ab42f9b is 50, key is test_row_0/B:col10/1732052576220/Put/seqid=0 2024-11-19T21:42:57,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742448_1624 (size=12301) 2024-11-19T21:42:57,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/8321639ed06d4bb2bc83498f9ab42f9b 2024-11-19T21:42:57,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/62508f5df40344c1badd8299917dc4ed is 50, key is test_row_0/C:col10/1732052576220/Put/seqid=0 2024-11-19T21:42:57,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742449_1625 (size=12301) 2024-11-19T21:42:57,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/62508f5df40344c1badd8299917dc4ed 2024-11-19T21:42:57,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/29de3a29b5fd4c2389b3819688694aa6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/29de3a29b5fd4c2389b3819688694aa6 2024-11-19T21:42:57,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/29de3a29b5fd4c2389b3819688694aa6, entries=150, sequenceid=267, filesize=30.5 K 2024-11-19T21:42:57,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/8321639ed06d4bb2bc83498f9ab42f9b as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/8321639ed06d4bb2bc83498f9ab42f9b 2024-11-19T21:42:57,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/8321639ed06d4bb2bc83498f9ab42f9b, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T21:42:57,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/62508f5df40344c1badd8299917dc4ed as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/62508f5df40344c1badd8299917dc4ed 2024-11-19T21:42:57,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/62508f5df40344c1badd8299917dc4ed, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T21:42:57,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 8b95a52d5be5c4d97b13c59880275ce5 in 1727ms, sequenceid=267, compaction requested=false 2024-11-19T21:42:57,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b95a52d5be5c4d97b13c59880275ce5: 2024-11-19T21:43:04,173 INFO [Thread-2464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2464. 2024-11-19T21:43:04,173 DEBUG [Thread-2464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:43:04,173 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,173 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:43:04,180 INFO [Thread-2460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2460. 2024-11-19T21:43:04,180 DEBUG [Thread-2460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:43:04,180 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,180 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:43:04,189 INFO [Thread-2466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2466. 2024-11-19T21:43:04,189 DEBUG [Thread-2466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:43:04,189 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,189 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:43:04,211 INFO [Thread-2468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2468. 2024-11-19T21:43:04,211 DEBUG [Thread-2468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T21:43:04,211 DEBUG [Thread-2468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,211 DEBUG [Thread-2468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,211 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:43:04,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4296 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4019 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4017 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4285 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4008 2024-11-19T21:43:04,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T21:43:04,212 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:43:04,212 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:43:04,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:04,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:43:04,214 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T21:43:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T21:43:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T21:43:04,220 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052584219"}]},"ts":"1732052584219"} 2024-11-19T21:43:04,221 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T21:43:04,221 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T21:43:04,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T21:43:04,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, UNASSIGN}] 2024-11-19T21:43:04,225 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, UNASSIGN 2024-11-19T21:43:04,226 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=8b95a52d5be5c4d97b13c59880275ce5, regionState=CLOSING, regionLocation=707262ffa86d,42821,1732052394005 2024-11-19T21:43:04,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, UNASSIGN because future has completed 2024-11-19T21:43:04,228 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T21:43:04,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005}] 2024-11-19T21:43:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T21:43:04,381 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:04,381 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T21:43:04,381 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 8b95a52d5be5c4d97b13c59880275ce5, disabling compactions & flushes 2024-11-19T21:43:04,381 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:43:04,381 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:43:04,382 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. after waiting 0 ms 2024-11-19T21:43:04,382 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:43:04,382 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 8b95a52d5be5c4d97b13c59880275ce5 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-19T21:43:04,391 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195f32056585654e5ca371ec8ba45a700e_8b95a52d5be5c4d97b13c59880275ce5 is 50, key is test_row_0/A:col10/1732052584188/Put/seqid=0 2024-11-19T21:43:04,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742450_1626 (size=9914) 2024-11-19T21:43:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T21:43:04,796 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T21:43:04,804 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195f32056585654e5ca371ec8ba45a700e_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195f32056585654e5ca371ec8ba45a700e_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:04,806 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/0228c544661849ee97f3d171711a85a1, store: [table=TestAcidGuarantees family=A region=8b95a52d5be5c4d97b13c59880275ce5] 2024-11-19T21:43:04,807 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/0228c544661849ee97f3d171711a85a1 is 175, key is test_row_0/A:col10/1732052584188/Put/seqid=0 2024-11-19T21:43:04,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742451_1627 (size=22561) 2024-11-19T21:43:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T21:43:05,214 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/0228c544661849ee97f3d171711a85a1 2024-11-19T21:43:05,226 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e343137baf2e46cb92b881b3eab3717f is 50, key is test_row_0/B:col10/1732052584188/Put/seqid=0 2024-11-19T21:43:05,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742452_1628 (size=9857) 2024-11-19T21:43:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T21:43:05,631 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e343137baf2e46cb92b881b3eab3717f 2024-11-19T21:43:05,642 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/5b1650cec51f4c98af7f3c25574ba35f is 50, key is test_row_0/C:col10/1732052584188/Put/seqid=0 2024-11-19T21:43:05,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742453_1629 (size=9857) 2024-11-19T21:43:06,048 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/5b1650cec51f4c98af7f3c25574ba35f 2024-11-19T21:43:06,060 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/A/0228c544661849ee97f3d171711a85a1 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/0228c544661849ee97f3d171711a85a1 2024-11-19T21:43:06,065 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/0228c544661849ee97f3d171711a85a1, entries=100, sequenceid=277, filesize=22.0 K 2024-11-19T21:43:06,067 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/B/e343137baf2e46cb92b881b3eab3717f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e343137baf2e46cb92b881b3eab3717f 2024-11-19T21:43:06,069 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e343137baf2e46cb92b881b3eab3717f, entries=100, sequenceid=277, filesize=9.6 K 2024-11-19T21:43:06,070 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/.tmp/C/5b1650cec51f4c98af7f3c25574ba35f as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/5b1650cec51f4c98af7f3c25574ba35f 2024-11-19T21:43:06,073 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/5b1650cec51f4c98af7f3c25574ba35f, entries=100, sequenceid=277, filesize=9.6 K 2024-11-19T21:43:06,073 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8b95a52d5be5c4d97b13c59880275ce5 in 1691ms, sequenceid=277, compaction requested=true 2024-11-19T21:43:06,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/342ac102b61a4986bd84de2c1f237adc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/ae78f1877c3b4c3db1c7fe967ad7cc94, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/34673efba659475e9b54f69feee55ca8, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a6cc7e5790b64697b452017ff28b8aac, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4194529ba33644dfb0d82a4f90637ca5, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/e724dcdcbbad45f49a59602baa39882e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/de74b4a3ae9e4ca9882861926fbf5863, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b181184f474442afbe25524842bdd456, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/988a29ad71c14ee4b54571b6175e7644, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b6b5964eee9a4488b067e4ffb37bd119, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4998537b9ad9492db03540375f013343, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/5e9dfe67ff2d49a89e1b5195c3ea793c, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3f205c02f5a147cfbf6b5091c9e7ea46, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/69398b0e5338406184bb88e1756b86ab, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3b390c8a226b4fc1863a6ca6b10a0ade, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/d156df28647145d0842079056608249b] to archive 2024-11-19T21:43:06,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:43:06,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/342ac102b61a4986bd84de2c1f237adc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/342ac102b61a4986bd84de2c1f237adc 2024-11-19T21:43:06,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/ae78f1877c3b4c3db1c7fe967ad7cc94 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/ae78f1877c3b4c3db1c7fe967ad7cc94 2024-11-19T21:43:06,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/34673efba659475e9b54f69feee55ca8 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/34673efba659475e9b54f69feee55ca8 2024-11-19T21:43:06,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a6cc7e5790b64697b452017ff28b8aac to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a6cc7e5790b64697b452017ff28b8aac 2024-11-19T21:43:06,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4194529ba33644dfb0d82a4f90637ca5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4194529ba33644dfb0d82a4f90637ca5 2024-11-19T21:43:06,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/e724dcdcbbad45f49a59602baa39882e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/e724dcdcbbad45f49a59602baa39882e 2024-11-19T21:43:06,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/de74b4a3ae9e4ca9882861926fbf5863 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/de74b4a3ae9e4ca9882861926fbf5863 2024-11-19T21:43:06,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b181184f474442afbe25524842bdd456 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b181184f474442afbe25524842bdd456 2024-11-19T21:43:06,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/988a29ad71c14ee4b54571b6175e7644 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/988a29ad71c14ee4b54571b6175e7644 2024-11-19T21:43:06,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b6b5964eee9a4488b067e4ffb37bd119 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/b6b5964eee9a4488b067e4ffb37bd119 2024-11-19T21:43:06,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4998537b9ad9492db03540375f013343 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/4998537b9ad9492db03540375f013343 2024-11-19T21:43:06,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/5e9dfe67ff2d49a89e1b5195c3ea793c to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/5e9dfe67ff2d49a89e1b5195c3ea793c 2024-11-19T21:43:06,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3f205c02f5a147cfbf6b5091c9e7ea46 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3f205c02f5a147cfbf6b5091c9e7ea46 2024-11-19T21:43:06,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/69398b0e5338406184bb88e1756b86ab to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/69398b0e5338406184bb88e1756b86ab 2024-11-19T21:43:06,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3b390c8a226b4fc1863a6ca6b10a0ade to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/3b390c8a226b4fc1863a6ca6b10a0ade 2024-11-19T21:43:06,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/d156df28647145d0842079056608249b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/d156df28647145d0842079056608249b 2024-11-19T21:43:06,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/14b6a0db9d0d49b3baf28645590f017e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d07ac4ca2c2143aa8bafb2062841c858, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e2d4f8614943467ba215b8df167e7e05, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/90ac60c4747d4b44b04a7c1dc73bcb23, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/8fdc1292e92644378965db43440fd614, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e3488a0f18014000b603830c3ae24b9e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d8250395c351458b9ffd377ca4ed8de1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/84c2b353d4a44a8789d9c854db01c57d, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/7e4cc7d473da41c7bb8fa4369ce66a1e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e91dd5ee818a4aaaa3647dac4100f334, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/3d6c1180ace949de9b00101ee2aac079, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/66b474a10df7441da65ef001f6473117, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/184ba5bc253b475893fb65158a945aa7, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e64f7034d60648f5acf2fb52f09fb4f0, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/bb3e21bc1d5e4d319a4ba26d8454cc06, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/a97544fbd5cc4129ad35a5247cabd74d] to archive 2024-11-19T21:43:06,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:43:06,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/14b6a0db9d0d49b3baf28645590f017e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/14b6a0db9d0d49b3baf28645590f017e 2024-11-19T21:43:06,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d07ac4ca2c2143aa8bafb2062841c858 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d07ac4ca2c2143aa8bafb2062841c858 2024-11-19T21:43:06,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e2d4f8614943467ba215b8df167e7e05 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e2d4f8614943467ba215b8df167e7e05 2024-11-19T21:43:06,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/90ac60c4747d4b44b04a7c1dc73bcb23 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/90ac60c4747d4b44b04a7c1dc73bcb23 2024-11-19T21:43:06,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/8fdc1292e92644378965db43440fd614 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/8fdc1292e92644378965db43440fd614 2024-11-19T21:43:06,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e3488a0f18014000b603830c3ae24b9e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e3488a0f18014000b603830c3ae24b9e 2024-11-19T21:43:06,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d8250395c351458b9ffd377ca4ed8de1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/d8250395c351458b9ffd377ca4ed8de1 2024-11-19T21:43:06,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/84c2b353d4a44a8789d9c854db01c57d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/84c2b353d4a44a8789d9c854db01c57d 2024-11-19T21:43:06,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/7e4cc7d473da41c7bb8fa4369ce66a1e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/7e4cc7d473da41c7bb8fa4369ce66a1e 2024-11-19T21:43:06,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e91dd5ee818a4aaaa3647dac4100f334 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e91dd5ee818a4aaaa3647dac4100f334 2024-11-19T21:43:06,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/3d6c1180ace949de9b00101ee2aac079 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/3d6c1180ace949de9b00101ee2aac079 2024-11-19T21:43:06,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/66b474a10df7441da65ef001f6473117 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/66b474a10df7441da65ef001f6473117 2024-11-19T21:43:06,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/184ba5bc253b475893fb65158a945aa7 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/184ba5bc253b475893fb65158a945aa7 2024-11-19T21:43:06,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e64f7034d60648f5acf2fb52f09fb4f0 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e64f7034d60648f5acf2fb52f09fb4f0 2024-11-19T21:43:06,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/bb3e21bc1d5e4d319a4ba26d8454cc06 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/bb3e21bc1d5e4d319a4ba26d8454cc06 2024-11-19T21:43:06,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/a97544fbd5cc4129ad35a5247cabd74d to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/a97544fbd5cc4129ad35a5247cabd74d 2024-11-19T21:43:06,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a29cec471bc04747884887db2fb3fcea, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8a090ebd7a6a4c94ba2c2b4c02d662a2, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a49a94d4f23b4912ad33b1d79c22c7d4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6f819d824e8a4b39ac5077cefbd7f121, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6d854027df404d419a55907b6c5250b1, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/3f44cfe6e02440aab5004026d5691e71, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/92f284763a864407966296df3c96e05b, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d159cfaa69f64078b3c2ab04751ec39e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/eeae3065a46344ad9f175d322f7f1008, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8e71cea084fa4c6e86cc49dedba871a4, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/27fe02b455504662a41fe1c1efab3714, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/bf5a40fd0a16484f89f71a5def0a0753, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/b4a9f03d5b4d40c4a5a847f11f7c21fc, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d16e102b930d4de0a0a804af7950a594, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/adfa111661e140a486f1c95996e9050e, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/18a790d0356d4088a2dbe9f0bc23d7a3] to archive 2024-11-19T21:43:06,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T21:43:06,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a29cec471bc04747884887db2fb3fcea to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a29cec471bc04747884887db2fb3fcea 2024-11-19T21:43:06,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8a090ebd7a6a4c94ba2c2b4c02d662a2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8a090ebd7a6a4c94ba2c2b4c02d662a2 2024-11-19T21:43:06,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a49a94d4f23b4912ad33b1d79c22c7d4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/a49a94d4f23b4912ad33b1d79c22c7d4 2024-11-19T21:43:06,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6f819d824e8a4b39ac5077cefbd7f121 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6f819d824e8a4b39ac5077cefbd7f121 2024-11-19T21:43:06,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6d854027df404d419a55907b6c5250b1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/6d854027df404d419a55907b6c5250b1 2024-11-19T21:43:06,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/3f44cfe6e02440aab5004026d5691e71 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/3f44cfe6e02440aab5004026d5691e71 2024-11-19T21:43:06,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/92f284763a864407966296df3c96e05b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/92f284763a864407966296df3c96e05b 2024-11-19T21:43:06,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d159cfaa69f64078b3c2ab04751ec39e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d159cfaa69f64078b3c2ab04751ec39e 2024-11-19T21:43:06,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/eeae3065a46344ad9f175d322f7f1008 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/eeae3065a46344ad9f175d322f7f1008 2024-11-19T21:43:06,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8e71cea084fa4c6e86cc49dedba871a4 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/8e71cea084fa4c6e86cc49dedba871a4 2024-11-19T21:43:06,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/27fe02b455504662a41fe1c1efab3714 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/27fe02b455504662a41fe1c1efab3714 2024-11-19T21:43:06,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/bf5a40fd0a16484f89f71a5def0a0753 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/bf5a40fd0a16484f89f71a5def0a0753 2024-11-19T21:43:06,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/b4a9f03d5b4d40c4a5a847f11f7c21fc to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/b4a9f03d5b4d40c4a5a847f11f7c21fc 2024-11-19T21:43:06,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d16e102b930d4de0a0a804af7950a594 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/d16e102b930d4de0a0a804af7950a594 2024-11-19T21:43:06,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/adfa111661e140a486f1c95996e9050e to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/adfa111661e140a486f1c95996e9050e 2024-11-19T21:43:06,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/18a790d0356d4088a2dbe9f0bc23d7a3 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/18a790d0356d4088a2dbe9f0bc23d7a3 2024-11-19T21:43:06,116 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/recovered.edits/280.seqid, newMaxSeqId=280, maxSeqId=4 2024-11-19T21:43:06,116 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. 2024-11-19T21:43:06,116 DEBUG [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 8b95a52d5be5c4d97b13c59880275ce5: Waiting for close lock at 1732052584381Running coprocessor pre-close hooks at 1732052584381Disabling compacts and flushes for region at 1732052584381Disabling writes for close at 1732052584382 (+1 ms)Obtaining lock to block concurrent updates at 1732052584382Preparing flush snapshotting stores in 8b95a52d5be5c4d97b13c59880275ce5 at 1732052584382Finished memstore snapshotting TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732052584383 (+1 ms)Flushing stores of TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5. at 1732052584384 (+1 ms)Flushing 8b95a52d5be5c4d97b13c59880275ce5/A: creating writer at 1732052584385 (+1 ms)Flushing mob file 8b95a52d5be5c4d97b13c59880275ce5/A: appending metadata at 1732052584390 (+5 ms)Flushing mob file 8b95a52d5be5c4d97b13c59880275ce5/A: closing flushed file at 1732052584390Flushing 8b95a52d5be5c4d97b13c59880275ce5/A: appending metadata at 1732052584806 (+416 ms)Flushing 8b95a52d5be5c4d97b13c59880275ce5/A: closing flushed file at 1732052584807 (+1 ms)Flushing 8b95a52d5be5c4d97b13c59880275ce5/B: creating writer at 1732052585221 (+414 ms)Flushing 8b95a52d5be5c4d97b13c59880275ce5/B: appending metadata at 1732052585225 (+4 ms)Flushing 8b95a52d5be5c4d97b13c59880275ce5/B: closing flushed file at 1732052585225Flushing 8b95a52d5be5c4d97b13c59880275ce5/C: creating writer at 1732052585638 (+413 ms)Flushing 8b95a52d5be5c4d97b13c59880275ce5/C: appending metadata at 1732052585642 (+4 ms)Flushing 8b95a52d5be5c4d97b13c59880275ce5/C: closing flushed file at 1732052585642Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@be01ae4: reopening flushed file at 1732052586059 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ee4d0ec: reopening flushed file at 1732052586066 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2829d75e: reopening flushed file at 1732052586070 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8b95a52d5be5c4d97b13c59880275ce5 in 1691ms, sequenceid=277, compaction requested=true at 1732052586073 (+3 ms)Writing region close event to WAL at 1732052586113 (+40 ms)Running coprocessor post-close hooks at 1732052586116 (+3 ms)Closed at 1732052586116 2024-11-19T21:43:06,118 INFO [RS_CLOSE_REGION-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=8b95a52d5be5c4d97b13c59880275ce5, regionState=CLOSED 2024-11-19T21:43:06,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 because future has completed 2024-11-19T21:43:06,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-19T21:43:06,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 8b95a52d5be5c4d97b13c59880275ce5, server=707262ffa86d,42821,1732052394005 in 1.8920 sec 2024-11-19T21:43:06,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-19T21:43:06,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b95a52d5be5c4d97b13c59880275ce5, UNASSIGN in 1.8980 sec 2024-11-19T21:43:06,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T21:43:06,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9030 sec 2024-11-19T21:43:06,128 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732052586128"}]},"ts":"1732052586128"} 2024-11-19T21:43:06,129 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T21:43:06,129 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T21:43:06,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9160 sec 2024-11-19T21:43:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T21:43:06,370 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:43:06,371 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T21:43:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:43:06,376 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:43:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T21:43:06,379 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:43:06,381 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,384 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C, FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/recovered.edits] 2024-11-19T21:43:06,385 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/0228c544661849ee97f3d171711a85a1 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/0228c544661849ee97f3d171711a85a1 2024-11-19T21:43:06,386 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/29de3a29b5fd4c2389b3819688694aa6 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/29de3a29b5fd4c2389b3819688694aa6 2024-11-19T21:43:06,387 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a966a67be54e4d29aced0038e2c64773 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/A/a966a67be54e4d29aced0038e2c64773 2024-11-19T21:43:06,389 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/8321639ed06d4bb2bc83498f9ab42f9b to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/8321639ed06d4bb2bc83498f9ab42f9b 2024-11-19T21:43:06,390 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/b057b42faa9a4beeab699fd346ce29a2 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/b057b42faa9a4beeab699fd346ce29a2 2024-11-19T21:43:06,391 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e343137baf2e46cb92b881b3eab3717f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/B/e343137baf2e46cb92b881b3eab3717f 2024-11-19T21:43:06,393 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/5b1650cec51f4c98af7f3c25574ba35f to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/5b1650cec51f4c98af7f3c25574ba35f 2024-11-19T21:43:06,393 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/62508f5df40344c1badd8299917dc4ed to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/62508f5df40344c1badd8299917dc4ed 2024-11-19T21:43:06,395 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/f799e5fcf0f846e59483f0bde92578bf to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/C/f799e5fcf0f846e59483f0bde92578bf 2024-11-19T21:43:06,397 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/recovered.edits/280.seqid to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5/recovered.edits/280.seqid 2024-11-19T21:43:06,398 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/default/TestAcidGuarantees/8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,398 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T21:43:06,398 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T21:43:06,399 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T21:43:06,402 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119087227ac03724af5a0ceb1531d1f9851_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119087227ac03724af5a0ceb1531d1f9851_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,403 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111919696c7816334d55a30169e699ae928b_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111919696c7816334d55a30169e699ae928b_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,404 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111921afb220e32745a0ba70a52ec798756f_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111921afb220e32745a0ba70a52ec798756f_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,405 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111926b0e2e0e891477284a821c64bbbd20d_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111926b0e2e0e891477284a821c64bbbd20d_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,406 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193b54373fd46542e2912baada605d2fe2_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193b54373fd46542e2912baada605d2fe2_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,407 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194298ba069ed44a9a88a21ca3a6330fc8_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194298ba069ed44a9a88a21ca3a6330fc8_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,408 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194378c35f69984fb690f1bbac34d40ade_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194378c35f69984fb690f1bbac34d40ade_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,409 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ba1c3790bb24ba79e20baf72f82ab1a_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ba1c3790bb24ba79e20baf72f82ab1a_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,410 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195f32056585654e5ca371ec8ba45a700e_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195f32056585654e5ca371ec8ba45a700e_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,411 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119750117d495f94b6497de3f50f3b5d74e_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119750117d495f94b6497de3f50f3b5d74e_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,412 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198be69ae387314efea0527e9a4643c69a_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198be69ae387314efea0527e9a4643c69a_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,413 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e98e956c11e465d8bbbcf70b9085123_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e98e956c11e465d8bbbcf70b9085123_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,414 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b3d4ef714002491ab950ad9a6759d95c_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b3d4ef714002491ab950ad9a6759d95c_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,415 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b4e431581d364a4c9951efb532503b1f_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b4e431581d364a4c9951efb532503b1f_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,416 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119be9f65e3b5f242f6a00b28ef0bde6948_8b95a52d5be5c4d97b13c59880275ce5 to hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119be9f65e3b5f242f6a00b28ef0bde6948_8b95a52d5be5c4d97b13c59880275ce5 2024-11-19T21:43:06,417 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T21:43:06,419 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:43:06,421 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T21:43:06,423 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T21:43:06,425 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:43:06,425 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T21:43:06,425 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732052586425"}]},"ts":"9223372036854775807"} 2024-11-19T21:43:06,427 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T21:43:06,427 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8b95a52d5be5c4d97b13c59880275ce5, NAME => 'TestAcidGuarantees,,1732052552070.8b95a52d5be5c4d97b13c59880275ce5.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T21:43:06,427 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T21:43:06,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732052586427"}]},"ts":"9223372036854775807"} 2024-11-19T21:43:06,429 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T21:43:06,430 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T21:43:06,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-19T21:43:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T21:43:06,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T21:43:06,493 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T21:43:06,502 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238 (was 239), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=349 (was 368), ProcessCount=11 (was 11), AvailableMemoryMB=4596 (was 4799) 2024-11-19T21:43:06,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T21:43:06,503 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:43:06,503 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:43:06,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:06,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:06,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T21:43:06,503 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T21:43:06,503 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=840105357, stopped=false 2024-11-19T21:43:06,504 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=707262ffa86d,37541,1732052393169 2024-11-19T21:43:06,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T21:43:06,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T21:43:06,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:43:06,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:43:06,567 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T21:43:06,569 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:43:06,569 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T21:43:06,569 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T21:43:06,570 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:43:06,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:06,571 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '707262ffa86d,42821,1732052394005' ***** 2024-11-19T21:43:06,571 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T21:43:06,571 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T21:43:06,572 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T21:43:06,572 INFO [RS:0;707262ffa86d:42821 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T21:43:06,573 INFO [RS:0;707262ffa86d:42821 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T21:43:06,573 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(959): stopping server 707262ffa86d,42821,1732052394005 2024-11-19T21:43:06,573 INFO [RS:0;707262ffa86d:42821 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T21:43:06,573 INFO [RS:0;707262ffa86d:42821 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;707262ffa86d:42821. 2024-11-19T21:43:06,573 DEBUG [RS:0;707262ffa86d:42821 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T21:43:06,573 DEBUG [RS:0;707262ffa86d:42821 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:06,573 INFO [RS:0;707262ffa86d:42821 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T21:43:06,573 INFO [RS:0;707262ffa86d:42821 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T21:43:06,573 INFO [RS:0;707262ffa86d:42821 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T21:43:06,573 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T21:43:06,574 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T21:43:06,574 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T21:43:06,574 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:43:06,574 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T21:43:06,574 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T21:43:06,574 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T21:43:06,574 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T21:43:06,574 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T21:43:06,574 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T21:43:06,592 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/info/fce6aba483674d319d8ffa8eb2dd15e8 is 95, key is TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6./info:/1732052428286/DeleteFamily/seqid=0 2024-11-19T21:43:06,595 INFO [regionserver/707262ffa86d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:43:06,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742454_1630 (size=6514) 2024-11-19T21:43:06,774 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:43:06,975 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:43:06,997 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/info/fce6aba483674d319d8ffa8eb2dd15e8 2024-11-19T21:43:07,022 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/ns/56d9183579004b25b5f8eaec53f895c5 is 93, key is TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6./ns:/1732052428236/DeleteFamily/seqid=0 2024-11-19T21:43:07,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742455_1631 (size=5906) 2024-11-19T21:43:07,175 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:43:07,242 INFO [regionserver/707262ffa86d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T21:43:07,243 INFO [regionserver/707262ffa86d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T21:43:07,375 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:43:07,427 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/ns/56d9183579004b25b5f8eaec53f895c5 2024-11-19T21:43:07,454 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/rep_barrier/220856b5b8024ecc8b7569b20bdb3bb8 is 102, key is TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6./rep_barrier:/1732052428236/DeleteFamily/seqid=0 2024-11-19T21:43:07,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742456_1632 (size=6025) 2024-11-19T21:43:07,576 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:43:07,776 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:43:07,858 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/rep_barrier/220856b5b8024ecc8b7569b20bdb3bb8 2024-11-19T21:43:07,880 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/table/04c791d0eb25483289a265dc873243bc is 96, key is TestAcidGuarantees,,1732052398846.7f292f8e50ece577fd7cb587fbc66fc6./table:/1732052428236/DeleteFamily/seqid=0 2024-11-19T21:43:07,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742457_1633 (size=5936) 2024-11-19T21:43:07,976 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:43:08,176 DEBUG [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T21:43:08,285 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/table/04c791d0eb25483289a265dc873243bc 2024-11-19T21:43:08,294 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/info/fce6aba483674d319d8ffa8eb2dd15e8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/info/fce6aba483674d319d8ffa8eb2dd15e8 2024-11-19T21:43:08,298 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/info/fce6aba483674d319d8ffa8eb2dd15e8, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T21:43:08,299 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/ns/56d9183579004b25b5f8eaec53f895c5 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/ns/56d9183579004b25b5f8eaec53f895c5 2024-11-19T21:43:08,303 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/ns/56d9183579004b25b5f8eaec53f895c5, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T21:43:08,304 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/rep_barrier/220856b5b8024ecc8b7569b20bdb3bb8 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/rep_barrier/220856b5b8024ecc8b7569b20bdb3bb8 2024-11-19T21:43:08,308 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/rep_barrier/220856b5b8024ecc8b7569b20bdb3bb8, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T21:43:08,309 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/.tmp/table/04c791d0eb25483289a265dc873243bc as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/table/04c791d0eb25483289a265dc873243bc 2024-11-19T21:43:08,313 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/table/04c791d0eb25483289a265dc873243bc, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T21:43:08,314 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false 2024-11-19T21:43:08,318 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T21:43:08,319 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T21:43:08,319 INFO [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T21:43:08,319 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732052586574Running coprocessor pre-close hooks at 1732052586574Disabling compacts and flushes for region at 1732052586574Disabling writes for close at 1732052586574Obtaining lock to block concurrent updates at 1732052586574Preparing flush snapshotting stores in 1588230740 at 1732052586574Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732052586575 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732052586575Flushing 1588230740/info: creating writer at 1732052586575Flushing 1588230740/info: appending metadata at 1732052586592 (+17 ms)Flushing 1588230740/info: closing flushed file at 1732052586592Flushing 1588230740/ns: creating writer at 1732052587006 (+414 ms)Flushing 1588230740/ns: appending metadata at 1732052587022 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732052587022Flushing 1588230740/rep_barrier: creating writer at 1732052587437 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732052587453 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732052587453Flushing 1588230740/table: creating writer at 1732052587864 (+411 ms)Flushing 1588230740/table: appending metadata at 1732052587880 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732052587880Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45a8c1ba: reopening flushed file at 1732052588293 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10f8d68: reopening flushed file at 1732052588299 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39d83989: reopening flushed file at 1732052588304 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d5937e0: reopening flushed file at 1732052588308 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false at 1732052588314 (+6 ms)Writing region close event to WAL at 1732052588315 (+1 ms)Running coprocessor post-close hooks at 1732052588318 (+3 ms)Closed at 1732052588319 (+1 ms) 2024-11-19T21:43:08,319 DEBUG [RS_CLOSE_META-regionserver/707262ffa86d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T21:43:08,377 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(976): stopping server 707262ffa86d,42821,1732052394005; all regions closed. 2024-11-19T21:43:08,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741834_1010 (size=25344) 2024-11-19T21:43:08,385 DEBUG [RS:0;707262ffa86d:42821 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/oldWALs 2024-11-19T21:43:08,386 INFO [RS:0;707262ffa86d:42821 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 707262ffa86d%2C42821%2C1732052394005.meta:.meta(num 1732052398359) 2024-11-19T21:43:08,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741832_1008 (size=14579871) 2024-11-19T21:43:08,390 DEBUG [RS:0;707262ffa86d:42821 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/oldWALs 2024-11-19T21:43:08,391 INFO [RS:0;707262ffa86d:42821 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 707262ffa86d%2C42821%2C1732052394005:(num 1732052397351) 2024-11-19T21:43:08,391 DEBUG [RS:0;707262ffa86d:42821 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T21:43:08,391 INFO [RS:0;707262ffa86d:42821 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T21:43:08,391 INFO [RS:0;707262ffa86d:42821 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T21:43:08,391 INFO [RS:0;707262ffa86d:42821 {}] hbase.ChoreService(370): Chore service for: regionserver/707262ffa86d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T21:43:08,391 INFO [RS:0;707262ffa86d:42821 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T21:43:08,391 INFO [regionserver/707262ffa86d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T21:43:08,392 INFO [RS:0;707262ffa86d:42821 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42821 2024-11-19T21:43:08,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/707262ffa86d,42821,1732052394005 2024-11-19T21:43:08,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T21:43:08,449 INFO [RS:0;707262ffa86d:42821 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T21:43:08,460 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [707262ffa86d,42821,1732052394005] 2024-11-19T21:43:08,467 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/707262ffa86d,42821,1732052394005 already deleted, retry=false 2024-11-19T21:43:08,467 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 707262ffa86d,42821,1732052394005 expired; onlineServers=0 2024-11-19T21:43:08,468 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '707262ffa86d,37541,1732052393169' ***** 2024-11-19T21:43:08,468 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T21:43:08,468 INFO [M:0;707262ffa86d:37541 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T21:43:08,468 INFO [M:0;707262ffa86d:37541 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T21:43:08,468 DEBUG [M:0;707262ffa86d:37541 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T21:43:08,468 DEBUG [M:0;707262ffa86d:37541 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T21:43:08,468 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T21:43:08,468 DEBUG [master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.small.0-1732052397018 {}] cleaner.HFileCleaner(306): Exit Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.small.0-1732052397018,5,FailOnTimeoutGroup] 2024-11-19T21:43:08,468 DEBUG [master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.large.0-1732052397017 {}] cleaner.HFileCleaner(306): Exit Thread[master/707262ffa86d:0:becomeActiveMaster-HFileCleaner.large.0-1732052397017,5,FailOnTimeoutGroup] 2024-11-19T21:43:08,468 INFO [M:0;707262ffa86d:37541 {}] hbase.ChoreService(370): Chore service for: master/707262ffa86d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T21:43:08,469 INFO [M:0;707262ffa86d:37541 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T21:43:08,469 DEBUG [M:0;707262ffa86d:37541 {}] master.HMaster(1795): Stopping service threads 2024-11-19T21:43:08,469 INFO [M:0;707262ffa86d:37541 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T21:43:08,469 INFO [M:0;707262ffa86d:37541 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T21:43:08,469 ERROR [M:0;707262ffa86d:37541 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (205990495) connection to localhost/127.0.0.1:37961 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:37961,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T21:43:08,470 INFO [M:0;707262ffa86d:37541 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T21:43:08,470 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T21:43:08,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T21:43:08,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T21:43:08,476 DEBUG [M:0;707262ffa86d:37541 {}] zookeeper.ZKUtil(347): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T21:43:08,476 WARN [M:0;707262ffa86d:37541 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T21:43:08,477 INFO [M:0;707262ffa86d:37541 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/.lastflushedseqids 2024-11-19T21:43:08,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742458_1634 (size=72) 2024-11-19T21:43:08,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:43:08,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42821-0x1015653ac4f0001, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:43:08,561 INFO [RS:0;707262ffa86d:42821 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T21:43:08,561 INFO [RS:0;707262ffa86d:42821 {}] regionserver.HRegionServer(1031): Exiting; stopping=707262ffa86d,42821,1732052394005; zookeeper connection closed. 2024-11-19T21:43:08,561 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21fa9f59 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21fa9f59 2024-11-19T21:43:08,562 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T21:43:08,889 INFO [M:0;707262ffa86d:37541 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T21:43:08,889 INFO [M:0;707262ffa86d:37541 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T21:43:08,890 DEBUG [M:0;707262ffa86d:37541 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T21:43:08,890 INFO [M:0;707262ffa86d:37541 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:43:08,890 DEBUG [M:0;707262ffa86d:37541 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:43:08,890 DEBUG [M:0;707262ffa86d:37541 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T21:43:08,890 DEBUG [M:0;707262ffa86d:37541 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:43:08,891 INFO [M:0;707262ffa86d:37541 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=751.19 KB heapSize=923.42 KB 2024-11-19T21:43:08,913 DEBUG [M:0;707262ffa86d:37541 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a07bd8b3e6114b8ba92da1beb51467a6 is 82, key is hbase:meta,,1/info:regioninfo/1732052398444/Put/seqid=0 2024-11-19T21:43:08,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742459_1635 (size=5672) 2024-11-19T21:43:09,317 INFO [M:0;707262ffa86d:37541 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2125 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a07bd8b3e6114b8ba92da1beb51467a6 2024-11-19T21:43:09,348 DEBUG [M:0;707262ffa86d:37541 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/51f15106ff6b470daf40e4604960ec56 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1732052555161/Put/seqid=0 2024-11-19T21:43:09,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742460_1636 (size=43491) 2024-11-19T21:43:09,753 INFO [M:0;707262ffa86d:37541 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750.64 KB at sequenceid=2125 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/51f15106ff6b470daf40e4604960ec56 2024-11-19T21:43:09,761 INFO [M:0;707262ffa86d:37541 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 51f15106ff6b470daf40e4604960ec56 2024-11-19T21:43:09,780 DEBUG [M:0;707262ffa86d:37541 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/93af2b95cf6e4f59a150d101581519a7 is 69, key is 707262ffa86d,42821,1732052394005/rs:state/1732052397073/Put/seqid=0 2024-11-19T21:43:09,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073742461_1637 (size=5156) 2024-11-19T21:43:10,183 INFO [M:0;707262ffa86d:37541 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2125 (bloomFilter=true), to=hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/93af2b95cf6e4f59a150d101581519a7 2024-11-19T21:43:10,187 DEBUG [M:0;707262ffa86d:37541 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a07bd8b3e6114b8ba92da1beb51467a6 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a07bd8b3e6114b8ba92da1beb51467a6 2024-11-19T21:43:10,190 INFO [M:0;707262ffa86d:37541 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a07bd8b3e6114b8ba92da1beb51467a6, entries=8, sequenceid=2125, filesize=5.5 K 2024-11-19T21:43:10,190 DEBUG [M:0;707262ffa86d:37541 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/51f15106ff6b470daf40e4604960ec56 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/51f15106ff6b470daf40e4604960ec56 2024-11-19T21:43:10,194 INFO [M:0;707262ffa86d:37541 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 51f15106ff6b470daf40e4604960ec56 2024-11-19T21:43:10,194 INFO [M:0;707262ffa86d:37541 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/51f15106ff6b470daf40e4604960ec56, entries=166, sequenceid=2125, filesize=42.5 K 2024-11-19T21:43:10,194 DEBUG [M:0;707262ffa86d:37541 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/93af2b95cf6e4f59a150d101581519a7 as hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/93af2b95cf6e4f59a150d101581519a7 2024-11-19T21:43:10,197 INFO [M:0;707262ffa86d:37541 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37961/user/jenkins/test-data/dd11dc3e-9c04-c6b4-d049-4cbdd32eb71b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/93af2b95cf6e4f59a150d101581519a7, entries=1, sequenceid=2125, filesize=5.0 K 2024-11-19T21:43:10,198 INFO [M:0;707262ffa86d:37541 {}] regionserver.HRegion(3140): Finished flush of dataSize ~751.19 KB/769222, heapSize ~923.13 KB/945280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2125, compaction requested=false 2024-11-19T21:43:10,199 INFO [M:0;707262ffa86d:37541 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T21:43:10,199 DEBUG [M:0;707262ffa86d:37541 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732052588890Disabling compacts and flushes for region at 1732052588890Disabling writes for close at 1732052588890Obtaining lock to block concurrent updates at 1732052588891 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732052588891Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=769222, getHeapSize=945520, getOffHeapSize=0, getCellsCount=2316 at 1732052588891Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732052588892 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732052588892Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732052588912 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732052588912Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732052589327 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732052589348 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732052589348Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732052589761 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732052589779 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732052589779Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63a97e77: reopening flushed file at 1732052590186 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7436b6fe: reopening flushed file at 1732052590190 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73649efd: reopening flushed file at 1732052590194 (+4 ms)Finished flush of dataSize ~751.19 KB/769222, heapSize ~923.13 KB/945280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2125, compaction requested=false at 1732052590198 (+4 ms)Writing region close event to WAL at 1732052590199 (+1 ms)Closed at 1732052590199 2024-11-19T21:43:10,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741830_1006 (size=908530) 2024-11-19T21:43:10,204 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T21:43:10,204 INFO [M:0;707262ffa86d:37541 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T21:43:10,204 INFO [M:0;707262ffa86d:37541 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37541 2024-11-19T21:43:10,204 INFO [M:0;707262ffa86d:37541 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T21:43:10,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:43:10,368 INFO [M:0;707262ffa86d:37541 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T21:43:10,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37541-0x1015653ac4f0000, quorum=127.0.0.1:63776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T21:43:10,375 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@51084f0f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T21:43:10,378 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@370621d9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T21:43:10,378 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T21:43:10,379 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4828e07b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T21:43:10,379 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ff64ba0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/hadoop.log.dir/,STOPPED} 2024-11-19T21:43:10,382 WARN [BP-224541545-172.17.0.2-1732052389373 heartbeating to localhost/127.0.0.1:37961 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T21:43:10,382 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T21:43:10,383 WARN [BP-224541545-172.17.0.2-1732052389373 heartbeating to localhost/127.0.0.1:37961 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-224541545-172.17.0.2-1732052389373 (Datanode Uuid 805e117f-2c48-4e99-9963-f94e586ba0fe) service to localhost/127.0.0.1:37961 2024-11-19T21:43:10,383 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T21:43:10,386 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/cluster_462aee02-848d-d834-50a3-23faa7aec3bf/data/data1/current/BP-224541545-172.17.0.2-1732052389373 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T21:43:10,387 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/cluster_462aee02-848d-d834-50a3-23faa7aec3bf/data/data2/current/BP-224541545-172.17.0.2-1732052389373 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T21:43:10,387 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T21:43:10,394 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T21:43:10,395 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T21:43:10,395 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T21:43:10,395 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T21:43:10,395 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42ad2f5c-4028-1d88-36fc-4ccbc60d1ffd/hadoop.log.dir/,STOPPED} 2024-11-19T21:43:10,410 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T21:43:10,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down